atlas-init 0.8.1__py3-none-any.whl → 0.10.0__py3-none-any.whl

atlas_init/tf_ext/tf_dep.py

@@ -1,16 +1,18 @@
 from __future__ import annotations
 
+from functools import total_ordering
 import logging
 from collections import defaultdict
 from pathlib import Path
-from typing import Iterable, NamedTuple
+from threading import RLock
+from typing import Callable, Iterable, NamedTuple
 
 import pydot
 from ask_shell import ShellError, new_task, run_and_wait
 from ask_shell._run import stop_runs_and_pool
 from ask_shell.run_pool import run_pool
 from model_lib import Entity, dump
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, model_validator
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_fixed
 from typer import Typer
 from zero_3rdparty.file_utils import ensure_parents_write_text
@@ -57,13 +59,25 @@ def tf_dep_graph(
     example_dirs = get_example_directories(repo_path, skip_names)
     logger.info(f"example_dirs: \n{'\n'.join(str(d) for d in sorted(example_dirs))}")
     with new_task("Find terraform graphs", total=len(example_dirs)) as task:
-        atlas_graph = parse_graphs(example_dirs, task)
+        atlas_graph = create_atlas_graph(example_dirs, task)
     with new_task("Dump graph"):
         graph_yaml = atlas_graph.dump_yaml()
         ensure_parents_write_text(settings.atlas_graph_path, graph_yaml)
         logger.info(f"Atlas graph dumped to {settings.atlas_graph_path}")
 
 
+def create_atlas_graph(example_dirs: list[Path], task: new_task) -> AtlasGraph:
+    atlas_graph = AtlasGraph()
+
+    def on_graph(example_dir: Path, graph: pydot.Dot):
+        atlas_graph.add_edges(graph.get_edges())
+        atlas_graph.add_variable_edges(example_dir)
+
+    parse_graphs(on_graph, example_dirs, task)
+
+    return atlas_graph
+
+
 def print_edges(graph: pydot.Dot):
     edges = graph.get_edges()
     for edge in edges:
@@ -79,9 +93,15 @@ class ResourceParts(NamedTuple):
         return self.resource_type.split("_")[0]
 
 
+@total_ordering
 class ResourceRef(BaseModel):
     full_ref: str
 
+    @model_validator(mode="after")
+    def ensure_plain(self):
+        self.full_ref = plain_name(self.full_ref)
+        return self
+
     def _resource_parts(self) -> ResourceParts:
         match self.full_ref.split("."):
             case [resource_type, resource_name] if "_" in resource_type:
@@ -106,6 +126,11 @@ class ResourceRef(BaseModel):
     def is_module(self) -> bool:
         return self.full_ref.startswith(MODULE_PREFIX)
 
+    @property
+    def module_name(self) -> str:
+        assert self.is_module, f"ResourceRef {self.full_ref} is not a module"
+        return self.full_ref.removeprefix(MODULE_PREFIX).split(".")[0]
+
     @property
     def is_data(self) -> bool:
         return self.full_ref.startswith(DATA_PREFIX)
@@ -114,6 +139,22 @@ class ResourceRef(BaseModel):
     def resource_type(self) -> str:
         return self._resource_parts().resource_type
 
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, ResourceRef):
+            raise TypeError(f"cannot compare {type(self)} with {type(other)}")
+        return self.full_ref < other.full_ref
+
+    def __hash__(self) -> int:
+        return hash(self.full_ref)
+
+    def __eq__(self, other: object) -> bool:
+        if not isinstance(other, ResourceRef):
+            return NotImplemented
+        return self.full_ref == other.full_ref
+
+    def __str__(self) -> str:
+        return self.full_ref
+
 
 class EdgeParsed(BaseModel):
     parent: ResourceRef
@@ -149,7 +190,15 @@ class EdgeParsed(BaseModel):
 
 
 def edge_plain(edge_endpoint: pydot.EdgeEndpoint) -> str:
-    return str(edge_endpoint).strip('"').strip()
+    return plain_name(str(edge_endpoint))
+
+
+def node_plain(node: pydot.Node) -> str:
+    return plain_name(node.get_name())
+
+
+def plain_name(name: str) -> str:
+    return name.strip('"').strip()
 
 
 def edge_src_dest(edge: pydot.Edge) -> tuple[str, str]:
@@ -235,34 +284,27 @@ class AtlasGraph(Entity):
                         self.parent_child_edges[parent_type].add(child_type)
 
 
-def parse_graphs(example_dirs: list[Path], task: new_task, max_workers: int = 16, max_dirs: int = 9999) -> AtlasGraph:
-    atlas_graph = AtlasGraph()
+def parse_graphs(
+    on_graph: Callable[[Path, pydot.Dot], None], example_dirs: list[Path], task: new_task, max_dirs: int = 1_000
+) -> None:
     with run_pool("parse example graphs", total=len(example_dirs)) as executor:
         futures = {
             executor.submit(parse_graph, example_dir): example_dir
             for i, example_dir in enumerate(example_dirs)
             if i < max_dirs
         }
-        graphs = {}
-        for future in futures:
+        for future, example_dir in futures.items():
             try:
-                example_dir, graph_output = future.result()
+                _, graph = future.result()
             except ShellError as e:
-                logger.error(f"Error parsing graph for {futures[future]}: {e}")
+                logger.error(f"Error parsing graph for {example_dir}: {e}")
                 continue
             except KeyboardInterrupt:
                 logger.error("KeyboardInterrupt received, stopping graph parsing.")
                 stop_runs_and_pool("KeyboardInterrupt", immediate=True)
                 break
-            try:
-                graph = graphs[example_dir] = parse_graph_output(example_dir, graph_output)
-            except GraphParseError as e:
-                logger.error(e)
-                continue
-            atlas_graph.add_edges(graph.get_edges())
-            atlas_graph.add_variable_edges(example_dir)
+            on_graph(example_dir, graph)
             task.update(advance=1)
-    return atlas_graph
 
 
 class GraphParseError(Exception):
@@ -271,9 +313,13 @@ class GraphParseError(Exception):
         super().__init__(f"Failed to parse graph for {example_dir}: {message}")
 
 
+_lock = RLock()
+
+
 def parse_graph_output(example_dir: Path, graph_output: str, verbose: bool = False) -> pydot.Dot:
     assert graph_output, f"Graph output is empty for {example_dir}"
-    dots = pydot.graph_from_dot_data(graph_output)  # not thread safe, so we use the main thread here instead
+    with _lock:
+        dots = pydot.graph_from_dot_data(graph_output)
     if not dots:
         raise GraphParseError(example_dir, f"No graphs found in the output:\n{graph_output}")
     assert len(dots) == 1, f"Expected one graph for {example_dir}, got {len(dots)}"
@@ -297,10 +343,10 @@ class EmptyGraphOutputError(Exception):
 @retry(
     stop=stop_after_attempt(3),
     wait=wait_fixed(1),
-    retry=retry_if_exception_type(EmptyGraphOutputError),
+    retry=retry_if_exception_type((EmptyGraphOutputError, GraphParseError)),
     reraise=True,
 )
-def parse_graph(example_dir: Path) -> tuple[Path, str]:
+def parse_graph(example_dir: Path) -> tuple[Path, pydot.Dot]:
     env_vars = {
         "MONGODB_ATLAS_PREVIEW_PROVIDER_V2_ADVANCED_CLUSTER": "true" if is_v2_example_dir(example_dir) else "false",
     }
@@ -309,7 +355,8 @@ def parse_graph(example_dir: Path) -> tuple[Path, str]:
         run_and_wait("terraform init", cwd=example_dir, env=env_vars)
     run = run_and_wait("terraform graph", cwd=example_dir, env=env_vars)
     if graph_output := run.stdout_one_line:
-        return example_dir, graph_output
+        graph = parse_graph_output(example_dir, graph_output)  # just to make sure we get no errors
+        return example_dir, graph
     raise EmptyGraphOutputError(example_dir)
 
 

atlas_init/tf_ext/tf_example_readme.py

@@ -0,0 +1,392 @@
+from __future__ import annotations
+from collections import defaultdict
+from contextlib import suppress
+from functools import total_ordering
+import logging
+from pathlib import Path
+from typing import Callable, ClassVar, Iterable, Protocol, TypeAlias
+from ask_shell import new_task
+from ask_shell.rich_live import get_live_console
+from model_lib import Entity, parse_dict
+from pydantic import Field, model_validator
+import pydot
+from rich.tree import Tree
+import typer
+
+from atlas_init.settings.rich_utils import tree_text
+from atlas_init.tf_ext.gen_readme import ReadmeMarkers, generate_and_write_readme
+from atlas_init.tf_ext.models import EmojiCounter
+from atlas_init.tf_ext.models_module import README_FILENAME
+from atlas_init.tf_ext.tf_dep import EdgeParsed, ResourceRef, node_plain, parse_graph, parse_graphs
+
+logger = logging.getLogger(__name__)
+MODULES_JSON_RELATIVE_PATH = ".terraform/modules/modules.json"
+
+
+def tf_example_readme(
+    example_path: Path = typer.Option(
+        ..., "-e", "--example-path", help="Path to the example directory", default_factory=Path.cwd
+    ),
+    skip_module_details: list[str] = typer.Option(
+        ..., "-s", "--skip-module-details", help="List of module details to skip", default_factory=list
+    ),
+):
+    with new_task("parse example graph"):
+        _, example_graph_dot = parse_graph(example_path)  # ensures init is called
+        example_graph = ResourceGraph.from_graph(example_graph_dot)
+    with new_task("parse module graphs") as task:
+        modules_config = parse_modules_json(example_path, skip_module_details)
+        module_paths = modules_config.module_paths
+        module_graphs: dict[Path, ResourceGraph] = {}
+
+        def on_graph(example_dir: Path, graph: pydot.Dot):
+            module_graphs[example_dir] = ResourceGraph.from_graph(graph)
+
+        parse_graphs(on_graph, module_paths, task)
+    with new_task("create example module graph"):
+        # a graph when all resources in a module are treated as a single node.
+        modules_graph, emoji_counter = create_module_graph(example_graph)
+    with new_task(f"update {README_FILENAME}"):
+        modules_section = []
+        modules_trees_texts = []
+        module_dirs_used: set[Path] = set()
+
+        def add_module_tree(module_dir: Path):
+            # trees are only once per module, not per module instance
+            if module_dir in module_dirs_used:
+                return
+            module_dirs_used.add(module_dir)
+            module_graph = module_graphs[module_dir]
+            module_config = modules_config.get_by_path(module_dir)
+            emojis = ", ".join(emoji_counter.get_emoji(key) for key in module_config.keys)
+            if modules_config.skip_details(module_config):
+                tree = Tree(f"{module_dir.name} ({emojis})")
+                tree.add("details skipped")
+            else:
+                tree = module_graph.to_tree(f"{module_dir.name} ({emojis})", include_orphans=True)
+            get_live_console().print(tree)
+            modules_trees_texts.append(tree_text(tree))
+
+        for _, module_key in emoji_counter.emoji_name():
+            module_config = modules_config.get_by_key(module_key)
+            module_dir = module_config.absolute_path(example_path)
+            add_module_tree(module_dir)
+
+        def add_module_src(node: Tree, name: str) -> None:
+            config = modules_config.get_by_key(name)
+            node.add(f"{config.source}")
+
+        module_index_tree = emoji_tree(emoji_counter, tree_processor=add_module_src, name="Module Instances")
+        modules_section.extend(
+            [
+                "## Modules",
+                "",
+                "### Modules Instances",
+                "```sh",
+                tree_text(module_index_tree),
+                "```",
+                "### Module Definitions",
+                "",
+                "```sh",
+                "\n".join(modules_trees_texts),
+                "```",
+                "",
+                "### Graph with Dependencies",
+                "Any resource without a number prefix is defined at the root level.",
+                "",
+                as_mermaid(modules_graph),
+            ]
+        )
+        generators = ReadmeMarkers.readme_generators()
+        generators.insert(1, (ReadmeMarkers.MODULES, lambda _: "\n".join(modules_section)))
+        generate_and_write_readme(
+            example_path,
+            generators=generators,
+        )
+
+
+def create_module_graph(example_graph: ResourceGraph) -> tuple[ResourceGraph, EmojiCounter]:
+    emoji_counter = EmojiCounter()
+
+    def as_module_edge(parent: ResourceRef, child: ResourceRef) -> bool | ParentChild:
+        if not child.is_module:
+            return False
+        new_parent = add_emoji_prefix(parent, emoji_counter)
+        new_child = add_emoji_prefix(child, emoji_counter)
+        return new_parent, new_child
+
+    return create_subgraph(example_graph, as_module_edge), emoji_counter
+
+
+def emoji_tree(
+    counter: EmojiCounter, *, tree_processor: Callable[[Tree, str], None] | None = None, name: str = "Emoji Tree"
+) -> Tree:
+    tree = Tree(name)
+    for emoji, name in counter.emoji_name():
+        child = tree.add(f"{emoji}  {name}")
+        if tree_processor:
+            tree_processor(child, name)
+    return tree
+
+
+def add_emoji_prefix(ref: ResourceRef, emoji_counter: EmojiCounter) -> ResourceRef:
+    if ref.is_module:
+        return ResourceRef(full_ref=f"{emoji_counter.get_emoji(ref.module_name)} {ref.module_name}")
+    return ref
+
+
+def strip_emoji_prefix(ref: ResourceRef) -> ResourceRef:
+    old_ref = ref.full_ref
+    return ResourceRef(full_ref=old_ref.split(" ")[1]) if " " in old_ref else ref
+
+
+def as_module_name(ref: ResourceRef) -> str:
+    if ref.is_module:
+        return ref.module_name
+    return ""
+
+
+def as_module_ref(ref: ResourceRef) -> ResourceRef:
+    if name := as_module_name(ref):
+        return ResourceRef(full_ref=f"module.{name}")
+    return ref
+
+
+class _RootModuleIgnored(Exception):
+    pass
+
+
+ParentChild: TypeAlias = tuple[ResourceRef, ResourceRef]
+
+
+class ResourceGraph(Entity):
+    IGNORED_ORPHANS: ClassVar[set[str]] = {"node"}  # some extra output from `terraform graph` command
+    parent_children: dict[ResourceRef, set[ResourceRef]] = Field(default_factory=lambda: defaultdict(set))
+    children_parents: dict[ResourceRef, set[ResourceRef]] = Field(default_factory=lambda: defaultdict(set))
+    orphans: set[ResourceRef] = Field(default_factory=set)
+
+    @classmethod
+    def from_graph(cls, graph: pydot.Dot) -> "ResourceGraph":
+        resource_graph = cls()
+        resource_graph.add_edges(graph.get_edges())
+        for orphan in graph.get_node_list():
+            name = node_plain(orphan)
+            if name in cls.IGNORED_ORPHANS:
+                continue
+            ref = ResourceRef(full_ref=name)
+            resource_graph.add_orphan_if_not_found(ref)
+        return resource_graph
+
+    def add_orphan_if_not_found(self, orphan: ResourceRef):
+        if orphan not in self.parent_children and orphan not in self.children_parents:
+            self.orphans.add(orphan)
+
+    def add_edges(self, edges: list[pydot.Edge]):
+        for edge in edges:
+            parsed = EdgeParsed.from_edge(edge)
+            parent = parsed.parent
+            child = parsed.child
+            if str(parent) == ".this":
+                logger.info(f"parent: {parent} child: {child}")
+            self.add_edge(parent, child)
+
+    def add_edge(self, parent: ResourceRef, child: ResourceRef):
+        self.parent_children[parent].add(child)
+        self.children_parents[child].add(parent)
+
+    def all_edges(self) -> list[ParentChild]:
+        return [(parent, child) for parent in self.parent_children for child in self.parent_children[parent]]
+
+    @property
+    def all_parents(self) -> set[ResourceRef]:
+        return set(self.parent_children.keys())
+
+    def sorted_parents(self) -> Iterable[ResourceRef]:
+        used_parents = set()
+        remaining_parents = self.all_parents
+
+        def next_parent() -> ResourceRef | None:
+            candidates = [parent for parent in remaining_parents if not self.children_parents[parent] - used_parents]
+            return min(candidates) if candidates else None
+
+        while remaining_parents:
+            parent = next_parent()
+            if parent is None:
+                break
+            used_parents.add(parent)
+            yield parent
+            remaining_parents.remove(parent)
+
+    def to_tree(self, example_dir_name: str, include_orphans: bool = False) -> Tree:
+        root = Tree(example_dir_name)
+        trees: dict[ResourceRef, Tree] = {}
+
+        for parent in self.sorted_parents():
+            parent_tree = trees.setdefault(parent, Tree(parent.full_ref))
+            for child_ref in sorted(self.parent_children[parent]):
+                child_tree = trees.setdefault(child_ref, Tree(child_ref.full_ref))
+                parent_tree.add(child_tree)
+            if not self.children_parents[parent]:
+                root.add(parent_tree)
+        if include_orphans:
+            for orphan in sorted(self.orphans):
+                if orphan not in trees:
+                    root.add(Tree(orphan.full_ref))
+        return root
+
+
+def as_mermaid(graph: ResourceGraph) -> str:
+    nodes: dict[str, str] = {}
+
+    def mermaid_ref(ref: ResourceRef) -> str:
+        no_emoji = strip_emoji_prefix(ref)
+        mermaid = str(no_emoji)
+        assert '"' not in mermaid, f"mermaid ref should not contain quotes: {mermaid}"
+        nodes[mermaid] = str(ref)  # want original ref in label
+        return mermaid
+
+    def add_mermaid_edge(parent: ResourceRef, child: ResourceRef) -> str:
+        parent_ref = mermaid_ref(parent)
+        child_ref = mermaid_ref(child)
+        return f"{parent_ref} --> {child_ref}"
+
+    edges = [add_mermaid_edge(parent, child) for parent, child in graph.all_edges()]
+    return "\n".join(
+        [
+            "```mermaid",
+            "graph TD",
+            "    " + "\n    ".join(f'{mermaid}["{label}"]' for mermaid, label in sorted(nodes.items())),
+            "    " + "\n    ".join(sorted(edges)),
+            "```",
+        ]
+    )
+
+
+class EdgeFilter(Protocol):
+    def __call__(self, parent: ResourceRef, child: ResourceRef) -> bool | ParentChild: ...
+
+
+def create_subgraph(graph: ResourceGraph, edge_filter: EdgeFilter) -> ResourceGraph:
+    subgraph = ResourceGraph()
+    for parent in graph.sorted_parents():
+        for child in sorted(graph.parent_children[parent]):
+            filter_response = edge_filter(parent, child)
+            match filter_response:
+                case True:
+                    subgraph.add_edge(parent, child)
+                case False:
+                    continue
+                case (parent, child) if parent != child:
+                    subgraph.add_edge(parent, child)
+    return subgraph
+
+
+@total_ordering
+class ModuleExampleConfig(Entity):
+    keys: list[str]
+    rel_path: str = Field(alias="Dir", description="Relative path to the module example")
+    source: str = Field(
+        alias="Source",
+        description="Source of the module, for example: registry.terraform.io/terraform-aws-modules/vpc/aws",
+    )
+    version: str = Field(
+        alias="Version", description="Version of the module example, unset for local modules", default=""
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def move_key(cls, v: dict):
+        key = v.pop("Key", None)
+        if key:
+            v["keys"] = [key]
+        if v.get("Dir", "") == ".":
+            raise _RootModuleIgnored()
+        return v
+
+    @model_validator(mode="after")
+    def validate_keys(self):
+        if not self.keys:
+            raise ValueError("keys is required")
+        return self
+
+    @property
+    def key(self) -> str:
+        return ",".join(sorted(self.keys))
+
+    def absolute_path(self, example_path: Path) -> Path:
+        path = example_path / self.rel_path
+        if not path.exists():
+            raise ValueError(f"module path not found for {self.key}: {path}")
+        return path
+
+    def __lt__(self, other: object) -> bool:
+        if not isinstance(other, ModuleExampleConfig):
+            raise TypeError(f"cannot compare {type(self)} with {type(other)}")
+        return self.key < other.key
+
+
+class ModuleExampleConfigs(Entity):
+    example_path: Path
+    modules: dict[str, ModuleExampleConfig] = Field(default_factory=dict)
+    skip_module_details: set[str] = Field(default_factory=set)
+
+    @model_validator(mode="after")
+    def ensure_paths_exists(self):
+        not_exists: dict[str, Path] = {}
+        for config in self.modules.values():
+            path = config.absolute_path(self.example_path)
+            if not path.exists():
+                not_exists[config.key] = path
+        if not_exists:
+            raise ValueError(f"module paths not found: {not_exists}")
+        return self
+
+    @property
+    def module_paths(self) -> list[Path]:
+        return [config.absolute_path(self.example_path) for config in self.modules.values()]
+
+    def skip_details(self, config: ModuleExampleConfig) -> bool:
+        return any(key in self.skip_module_details for key in config.keys)
+
+    def get_by_path(self, module_dir: Path) -> ModuleExampleConfig:
+        for config in self.modules.values():
+            if config.absolute_path(self.example_path) == module_dir:
+                return config
+        raise ValueError(f"module not found for {module_dir}")
+
+    def get_by_key_or_none(self, key: str) -> ModuleExampleConfig | None:
+        return self.modules.get(key)
+
+    def get_by_key(self, key: str) -> ModuleExampleConfig:
+        return self.modules[key]
+
+    def modules_included(self, *, skip_keys: list[str]) -> list[ModuleExampleConfig]:
+        return [config for config in self.modules.values() if all(key not in skip_keys for key in config.keys)]
+
+    def add_module(self, config: ModuleExampleConfig):
+        key = config.keys[0]
+        assert len(config.keys) == 1, "only one key can be added at a time"
+        source = config.source
+        existing_config = next(
+            (existing_config for existing_config in self.modules.values() if source == existing_config.source),
+            None,
+        )
+        if existing_config:
+            existing_config.keys.append(key)
+            existing_config.keys.sort()
+            self.modules[key] = existing_config
+        else:
+            self.modules[key] = config
+
+
+def parse_modules_json(example_path: Path, skip_module_details: list[str] | None = None) -> ModuleExampleConfigs:
+    configs = ModuleExampleConfigs(example_path=example_path, skip_module_details=set(skip_module_details or []))
+    module_json_path = example_path / MODULES_JSON_RELATIVE_PATH
+    if not module_json_path.exists():
+        return configs
+    module_json = parse_dict(module_json_path)
+    for raw in module_json.get("Modules", []):
+        with suppress(_RootModuleIgnored):
+            config = ModuleExampleConfig(**raw)
+            configs.add_module(config)
+    return configs

atlas_init/tf_ext/tf_mod_gen.py

@@ -12,7 +12,7 @@ from zero_3rdparty.file_utils import clean_dir, copy, ensure_parents_write_text
 from atlas_init.cli_tf.example_update import UpdateExamples, update_examples
 from atlas_init.tf_ext.args import TF_CLI_CONFIG_FILE_ARG
 from atlas_init.tf_ext.gen_examples import generate_module_examples, read_example_dirs
-from atlas_init.tf_ext.gen_module_readme import generate_readme
+from atlas_init.tf_ext.gen_readme import generate_and_write_readme
 from atlas_init.tf_ext.gen_resource_main import generate_resource_main
 from atlas_init.tf_ext.gen_resource_output import generate_resource_output
 from atlas_init.tf_ext.gen_resource_variables import generate_module_variables
@@ -32,6 +32,7 @@ from atlas_init.tf_ext.plan_diffs import (
     read_variables_path,
 )
 from atlas_init.tf_ext.provider_schema import AtlasSchemaInfo, ResourceSchema, parse_atlas_schema
+from atlas_init.tf_ext.run_tf import validate_tf_workspace
 from atlas_init.tf_ext.schema_to_dataclass import convert_and_format
 from atlas_init.tf_ext.settings import TfExtSettings
 
@@ -131,29 +132,13 @@ def generate_resource_module(config: ModuleGenConfig, resource_type: str, atlas_
 def finalize_and_validate_module(config: ModuleGenConfig) -> Path:
     dump_versions_tf(config.module_out_path, skip_python=config.skip_python)
     logger.info(f"Module dumped to {config.module_out_path}, running checks")
-    validate_module(config.module_out_path, tf_cli_config_file=config.settings.tf_cli_config_file)
+    validate_tf_workspace(config.module_out_path, tf_cli_config_file=config.settings.tf_cli_config_file)
     return config.module_out_path
 
 
 OUT_BINARY_PATH = "tfplan.binary"
 
 
-def validate_module(tf_workdir: Path, *, tf_cli_config_file: Path | None = None):
-    terraform_commands = [
-        "terraform init",
-        "terraform fmt .",
-        "terraform validate .",
-    ]
-    env_extra = {}
-    if tf_cli_config_file:
-        env_extra["TF_CLI_CONFIG_FILE"] = str(tf_cli_config_file)
-    with new_task("Terraform Module Validate Checks", total=len(terraform_commands)) as task:
-        for command in terraform_commands:
-            attempts = 3 if command == "terraform init" else 1  # terraform init can fail due to network issues
-            run_and_wait(command, cwd=tf_workdir, env=env_extra, attempts=attempts)
-            task.update(advance=1)
-
-
 def module_examples_and_readme(config: ModuleGenConfig, *, example_var_file: Path | None = None) -> Path:
     path = config.module_out_path
     if (examples_test := config.examples_test_path) and examples_test.exists():
@@ -165,7 +150,7 @@ def module_examples_and_readme(config: ModuleGenConfig, *, example_var_file: Pat
         if examples_generated:
             with run_pool("Validating examples", total=len(examples_generated), exit_wait_timeout=60) as pool:
                 for example_path in examples_generated:
-                    pool.submit(validate_module, example_path)
+                    pool.submit(validate_tf_workspace, example_path)
 
     attribute_descriptions = parse_attribute_descriptions(config.settings)
     settings = config.settings
@@ -192,12 +177,10 @@ def module_examples_and_readme(config: ModuleGenConfig, *, example_var_file: Pat
     if out_event.changes:
         logger.info(f"Updated attribute descriptions: {len(out_event.changes)}")
         run_and_wait("terraform fmt -recursive .", cwd=path, ansi_content=False, allow_non_zero_exit=True)
-    if readme_path := config.readme_path():
-        with new_task("Generating README.md"):
-            readme_content = generate_readme(config)
-            ensure_parents_write_text(readme_path, readme_content)
+    with new_task("Generating README.md"):
+        generate_and_write_readme(config.module_out_path)
     if example_var_file:
-        examples = read_example_dirs(config.module_out_path)
+        examples = read_example_dirs(config.examples_path)
         if examples:
             failed_examples: list[Path] = []
             with run_pool("Running terraform plan on examples", total=len(examples), exit_wait_timeout=60) as pool:
@@ -227,7 +210,7 @@ def example_plan_checks(config: ModuleGenConfig, timeout_all_seconds: int = 60)
         with TemporaryDirectory() as temp_dir:
             stored_plan = Path(temp_dir) / "plan.json"
             tf_dir = config.example_path(check.example_name)
-            validate_module(tf_dir)
+            validate_tf_workspace(tf_dir)
             var_arg = f" -var-file={variables_path}" if variables_path else ""
             run_and_wait(f"terraform plan -out={OUT_BINARY_PATH}{var_arg}", cwd=tf_dir)
             run_and_wait(f"terraform show -json {OUT_BINARY_PATH} > {stored_plan}", cwd=tf_dir)

atlas_init/tf_ext/tf_modules.py

@@ -175,7 +175,9 @@ def tf_modules(
         logger.info(f"Missing modules: \n{'\n'.join(missing_resources)}")
 
 
-def generate_module_graphs(skipped_module_resource_types, settings, atlas_graph):
+def generate_module_graphs(
+    skipped_module_resource_types: Iterable[str], settings: TfExtSettings, atlas_graph: AtlasGraph
+):
     tree = Tree(
         "Module graphs",
     )