atlas-init 0.6.0__py3-none-any.whl → 0.8.0__py3-none-any.whl

atlas_init/tf_ext/gen_examples.py

@@ -0,0 +1,141 @@
+from contextlib import suppress
+from dataclasses import asdict
+from functools import singledispatch
+from pathlib import Path
+
+from zero_3rdparty import humps
+from zero_3rdparty.file_utils import clean_dir, ensure_parents_write_text
+
+from atlas_init.tf_ext.gen_resource_variables import generate_resource_variables
+from atlas_init.tf_ext.gen_versions import dump_versions_tf
+from atlas_init.tf_ext.models_module import ModuleGenConfig, ResourceAbs, ResourceGenConfig, ResourceTypePythonModule
+from atlas_init.tf_ext.py_gen import import_module_by_using_parents
+
+VARIABLE_PLACEHOLDER = "var."
+INDENT = "    "
+
+
+def _examples_casted(examples: dict) -> dict[str, ResourceAbs]:
+    return examples
+
+
+def read_example_dirs(module_path: Path) -> list[Path]:
+    return sorted(
+        example_dir
+        for example_dir in (module_path / "examples").glob("*")
+        if example_dir.is_dir()
+        and len(example_dir.name) > 2
+        and example_dir.name[:2].isdigit()
+        and (example_dir / "main.tf").exists()
+    )
+
+
+def generate_module_examples(
+    config: ModuleGenConfig,
+    module: ResourceTypePythonModule,
+    resource_type: str,
+    *,
+    skip_clean_dir: bool = False,
+) -> list[Path]:
+    test_path = config.examples_test_path
+    imported_module = import_module_by_using_parents(test_path)
+    examples = getattr(imported_module, "EXAMPLES")
+    assert isinstance(examples, dict), f"{imported_module} does not have an EXAMPLES attribute"
+    examples_parsed = _examples_casted(examples)
+    examples_generated: list[Path] = []
+    for example_name, example in examples_parsed.items():
+        dumped_resource = {k: v for k, v in asdict(example).items() if v is not None}
+        variables = {
+            k: f"{v}{k}"
+            for k, v in dumped_resource.items()
+            if isinstance(v, str) and v.startswith(VARIABLE_PLACEHOLDER)
+        }
+        dumped_resource |= variables
+        variable_names = set(variables.keys())
+        ignored_names = set(module.all_field_names) - variable_names
+        ignored_names |= module.all_skip_variables
+        resource_cls = module.resource_ext or module.resource
+        assert resource_cls, f"{module} does not have a resource class"
+        example_path = config.example_path(example_name)
+        if not skip_clean_dir and example_path.exists():
+            clean_dir(example_path)
+
+        variables_tf = generate_resource_variables(
+            resource_cls,
+            ResourceGenConfig(
+                name=resource_type, skip_variables_extra=ignored_names, required_variables=variable_names
+            ),
+        )
+        ensure_parents_write_text(example_path / "variables.tf", variables_tf)
+        variables_str = "\n".join(f"{k} = {dump_variable(v)}" for k, v in dumped_resource.items() if can_dump(v))
+        example_main = example_main_tf(config, variables_str)
+        ensure_parents_write_text(example_path / "main.tf", example_main)
+        dump_versions_tf(example_path, skip_python=config.skip_python, minimal=True)
+        examples_generated.append(example_path)
+    return examples_generated
+
+
+class NotDumpableError(Exception):
+    def __init__(self, value: object) -> None:
+        super().__init__(f"Cannot dump variable {value!r}")
+
+
+def can_dump(variable: object) -> bool:
+    with suppress(NotDumpableError):
+        dump_variable(variable)
+        return True
+    return False
+
+
+@singledispatch
+def dump_variable(variable: object) -> str:
+    raise NotDumpableError(variable)
+
+
+@dump_variable.register
+def dump_variable_str(variable: str) -> str:
+    if "." in variable:
+        return variable
+    return f'"{variable}"'
+
+
+@dump_variable.register
+def dump_variable_int(variable: int) -> str:
+    return str(variable)
+
+
+@dump_variable.register
+def dump_variable_float(variable: float) -> str:
+    return str(variable)
+
+
+@dump_variable.register
+def dump_variable_bool(variable: bool) -> str:
+    return "true" if variable else "false"
+
+
+@dump_variable.register
+def dump_variable_list(variable: list) -> str:
+    return f"[\n{', '.join(f'{INDENT}{dump_variable(nested)}' for nested in variable if can_dump(nested))}\n]"
+
+
+@dump_variable.register
+def dump_variable_set(variable: set) -> str:
+    return f"[\n{', '.join(f'{INDENT}{dump_variable(nested)}' for nested in variable if can_dump(nested))}\n]"
+
+
+@dump_variable.register
+def dump_variable_dict(variable: dict) -> str:
+    return "{\n" + "\n".join(f"{INDENT}{k} = {dump_variable(v)}" for k, v in variable.items() if can_dump(v)) + "\n}"
+
+
+def example_main_tf(module: ModuleGenConfig, variables: str) -> str:
+    variables_indented = "\n".join(f"{INDENT}{var}" for var in variables.split("\n"))
+    module_name_snake = humps.dekebabize(module.name)
+    return f"""\
+module "{module_name_snake}" {{
+    source = "../.."
+
+{variables_indented}
+}}
+"""

atlas_init/tf_ext/gen_module_readme.py

@@ -0,0 +1,131 @@
+import logging
+from enum import StrEnum
+
+from ask_shell import run_and_wait
+from zero_3rdparty.file_utils import ensure_parents_write_text, update_between_markers
+
+from atlas_init.tf_ext.gen_examples import read_example_dirs
+from atlas_init.tf_ext.models_module import ModuleGenConfig
+
+logger = logging.getLogger(__name__)
+_readme_disclaimer = """\
+## Disclaimer
+This Module is not meant for external consumption.
+It is part of a development PoC.
+Any usage problems will not be supported.
+However, if you have any ideas or feedback feel free to open a Github Issue!
+"""
+
+
+class ReadmeMarkers(StrEnum):
+    DISCLAIMER = "DISCLAIMER"
+    EXAMPLE = "TF_EXAMPLES"
+    TF_DOCS = "TF_DOCS"
+
+    @classmethod
+    def as_start(cls, marker_name: str) -> str:
+        return f"<!-- BEGIN_{marker_name} -->"
+
+    @classmethod
+    def as_end(cls, marker_name: str) -> str:
+        return f"<!-- END_{marker_name} -->"
+
+    @classmethod
+    def marker_lines(cls, marker_name: str) -> str:
+        return f"""\
+{cls.as_start(marker_name)}
+
+{cls.as_end(marker_name)}
+"""
+
+    @classmethod
+    def example_boilerplate(cls) -> str:
+        return "\n".join(cls.marker_lines(marker_name) for marker_name in list(cls))
+
+
+def read_examples(module: ModuleGenConfig) -> str:
+    example_dirs = read_example_dirs(module.module_out_path)
+    content = ["# Examples"]
+    for example_dir in example_dirs:
+        example_name = example_dir.name
+        header_name = example_name.replace("_", " ").replace("-", " ").title()
+        main_path = example_dir / "main.tf"
+        assert main_path.exists(), f"{main_path} does not exist, every example must have a main.tf"
+        content.extend(
+            [
+                f"## [{header_name}](./examples/{example_name})",
+                "",
+                "```terraform",
+                main_path.read_text(),
+                "```",
+                "",
+                "",
+            ]
+        )
+    return "\n".join(content)
+
+
+_static_terraform_config = """\
+formatter: markdown document
+output:
+  file: "FILENAME"
+  mode: inject
+  template: |-
+    START_MARKER
+    {{ .Content }}
+    END_MARKER
+sort:
+  enabled: true
+  by: required
+"""
+
+
+def terraform_docs_config_content(module: ModuleGenConfig) -> str:
+    config = _static_terraform_config
+    for replacement_in, replacement_out in [
+        ("FILENAME", module.readme_path().name),
+        ("START_MARKER", ReadmeMarkers.as_start(ReadmeMarkers.TF_DOCS)),
+        ("END_MARKER", ReadmeMarkers.as_end(ReadmeMarkers.TF_DOCS)),
+    ]:
+        config = config.replace(replacement_in, replacement_out)
+    return config
+
+
+def generate_readme(module: ModuleGenConfig) -> str:
+    readme_path = module.readme_path()
+    assert readme_path.exists(), (
+        f"{readme_path} does not exist, currently a boilerplate is expected, consider adding to {readme_path}\n{ReadmeMarkers.example_boilerplate()}"
+    )
+    update_between_markers(
+        readme_path,
+        _readme_disclaimer,
+        ReadmeMarkers.as_start(ReadmeMarkers.DISCLAIMER),
+        ReadmeMarkers.as_end(ReadmeMarkers.DISCLAIMER),
+    )
+    run_and_wait("terraform fmt -recursive .", cwd=module.module_out_path, allow_non_zero_exit=True, ansi_content=False)
+    example_section = read_examples(module)
+    update_between_markers(
+        readme_path,
+        example_section,
+        ReadmeMarkers.as_start(ReadmeMarkers.EXAMPLE),
+        ReadmeMarkers.as_end(ReadmeMarkers.EXAMPLE),
+    )
+    docs_config_path = module.terraform_docs_config_path()
+    if docs_config_path.exists():
+        logger.warning(f"{docs_config_path} already exists, skipping generation")
+    else:
+        config_content = terraform_docs_config_content(module)
+        ensure_parents_write_text(docs_config_path, config_content)
+        logger.info(f"generated {docs_config_path}")
+    run_and_wait(f"terraform-docs -c {docs_config_path} .", cwd=module.module_out_path)
+    readme_content = _default_link_updater(readme_path.read_text())
+    ensure_parents_write_text(readme_path, readme_content)
+    return readme_path.read_text()
+
+
+def _default_link_updater(readme_content: str) -> str:  # can be a global replacer for now
+    for replace_in, replace_out in {
+        "docs/resources/advanced_cluster": r"docs/resources/advanced_cluster%2520%2528preview%2520provider%25202.0.0%2529"
+    }.items():
+        readme_content = readme_content.replace(replace_in, replace_out)
+    return readme_content

atlas_init/tf_ext/gen_resource_main.py

@@ -0,0 +1,195 @@
+import logging
+from dataclasses import fields
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from typing import Iterable
+
+from ask_shell import run_and_wait
+
+from atlas_init.tf_ext.models_module import ModuleGenConfig, ResourceAbs, ResourceGenConfig
+from atlas_init.tf_ext.schema_to_dataclass import ResourceTypePythonModule
+
+logger = logging.getLogger(__name__)
+
+
+def local_name_varsx(resource_type: str) -> str:
+    return f"{resource_type}_varsx"
+
+
+def local_name_vars(resource_type: str) -> str:
+    return f"{resource_type}_vars"
+
+
+def locals_def(module: ResourceTypePythonModule) -> str:
+    base_defs = "\n".join(f"        {name} = var.{name}" for name in module.base_field_names_not_computed)
+    if extras := module.extra_fields_names:
+        extra_defs = "\n".join(f"        {name} = var.{name}" for name in extras)
+        base_def = f"    {local_name_varsx(module.resource_type)} = {{\n{base_defs}\n    }}"
+        extra_def = f"\n    {local_name_vars(module.resource_type)} = {{\n{extra_defs}\n    }}"
+    else:
+        base_def = f"    {local_name_vars(module.resource_type)} = {{\n{base_defs}\n    }}"
+        extra_def = ""
+    return f"""
+locals {{
+    {base_def}{extra_def}
+}}
+"""
+
+
+def data_external(module: ResourceTypePythonModule, config: ModuleGenConfig) -> str:
+    input_json_parts = [
+        f"local.{local_name_vars(module.resource_type)}",
+    ]
+    if module.extra_fields_names:
+        input_json_parts.append(f"local.{local_name_varsx(module.resource_type)}")
+    if extras := config.inputs_json_hcl_extras:
+        input_json_parts.extend(extras)
+    inputs_json_merge = input_json_parts[0] if len(input_json_parts) == 1 else f"merge({', '.join(input_json_parts)})"
+    return f"""
+data "external" "{module.resource_type}" {{
+    program = ["python3", "${{path.module}}/{module.resource_type}.py"]
+    query = {{
+        input_json = jsonencode({inputs_json_merge})
+    }}
+}}
+"""
+
+
+def resource_declare_direct(py_module: ResourceTypePythonModule, config: ResourceGenConfig) -> str:
+    parent_cls = py_module.resource
+    resource_type = py_module.resource_type
+    assert parent_cls, f"{resource_type} does not have a resource"
+    field_base = f"var.{resource_type}." if config.use_single_variable else "var."
+    field_values = "\n".join(
+        _field_value(parent_cls, name, field_base) for name in py_module.base_field_names_not_computed
+    )
+
+    return f"""
+resource "{py_module.resource_type}" "this" {{
+{field_values}
+}}
+"""
+
+
+def _field_value(parent_cls: type[ResourceAbs], field_name: str, field_base: str = "var.") -> str:
+    if ResourceAbs.is_computed_only(field_name, parent_cls):
+        return ""
+    this_indent = "  "
+    if ResourceAbs.is_block(field_name, parent_cls):
+        return "\n".join(f"{this_indent}{line}" for line in _handle_dynamic(parent_cls, field_name, field_base))
+    return this_indent + f"{field_name} = {field_base}{field_name}"
+
+
+def _handle_dynamic(
+    parent_cls: type[ResourceAbs], dynamic_field_name: str, existing_ref: str = "var."
+) -> Iterable[str]:
+    try:
+        container_type = next(
+            t for name, t in ResourceTypePythonModule.container_types(parent_cls) if name == dynamic_field_name
+        )
+    except StopIteration:
+        raise ValueError(f"Could not find container type for field {dynamic_field_name} in {parent_cls}")
+    hcl_ref = f"{dynamic_field_name}.value."
+    yield f'dynamic "{dynamic_field_name}" {{'
+    ref = existing_ref + dynamic_field_name
+    if container_type.is_list or container_type.is_set:
+        if container_type.is_optional:
+            yield f"  for_each = {ref} == null ? [] : {ref}"
+        else:
+            yield f"  for_each = {ref}"
+    elif container_type.is_dict:
+        raise NotImplementedError(f"Dict not supported for {dynamic_field_name} in {parent_cls}")
+    else:  # singular
+        if container_type.is_optional:
+            yield f"  for_each = {ref} == null ? [] : [{ref}]"
+        else:
+            yield f"  for_each = [{ref}]"
+    yield "  content {"
+    yield from [f"    {line}" for line in _nested_fields(container_type.type, hcl_ref)]
+    yield "  }"
+    yield "}"
+
+
+def _nested_fields(cls: type[ResourceAbs], hcl_ref: str) -> Iterable[str]:
+    for field in fields(cls):
+        field_name = field.name
+        if ResourceAbs.is_computed_only(field_name, cls):
+            continue
+        if ResourceAbs.is_block(field_name, cls):
+            yield from _handle_dynamic(cls, field_name, hcl_ref)
+        else:
+            yield _field_value(cls, field_name, hcl_ref)
+
+
+def resource_declare(
+    resource_type: str, required_fields: set[str], nested_fields: set[str], field_names: list[str]
+) -> str:
+    def output_field(field_name: str) -> str:
+        return f"data.external.{resource_type}.result.{field_name}"
+
+    def as_output_field(field_name: str) -> str:
+        if field_name in nested_fields:
+            if field_name in required_fields:
+                return f"jsondecode({output_field(field_name)})"
+            return f'{output_field(field_name)} == "" ? null : jsondecode({output_field(field_name)})'
+        if field_name in required_fields:
+            return output_field(field_name)
+        return f'{output_field(field_name)} == "" ? null : {output_field(field_name)}'
+
+    required = [f"    {field_name} = {as_output_field(field_name)}" for field_name in sorted(required_fields)]
+    non_required = [
+        f"    {field_name} = {as_output_field(field_name)}"
+        for field_name in sorted(field_names)
+        if field_name not in required_fields
+    ]
+    return f"""
+resource "{resource_type}" "this" {{
+    lifecycle {{
+        precondition {{
+            condition = length({output_field("error_message")}) == 0
+            error_message = {output_field("error_message")}
+        }}
+    }}
+
+{"\n".join(required)}
+{"\n".join(non_required)}
+}}
+"""
+
+
+def format_tf_content(content: str) -> str:
+    with TemporaryDirectory() as tmp_dir:
+        tmp_file = Path(tmp_dir) / "content.tf"
+        tmp_file.write_text(content)
+        try:
+            run_and_wait("terraform fmt .", cwd=tmp_dir)
+        except Exception as e:
+            logger.error(f"Failed to format tf content:\n{content}")
+            raise e
+        return tmp_file.read_text()
+
+
+def generate_resource_main(python_module: ResourceTypePythonModule, config: ModuleGenConfig) -> str:
+    resource = python_module.resource_ext or python_module.resource
+    assert resource, f"{python_module} does not have a resource"
+    resource_hcl = (
+        resource_declare_direct(python_module, config.resource_config(python_module.resource_type))
+        if config.skip_python
+        else resource_declare(
+            resource_type=python_module.resource_type,
+            required_fields=resource.REQUIRED_ATTRIBUTES,
+            nested_fields=resource.NESTED_ATTRIBUTES,
+            field_names=python_module.base_field_names_not_computed,
+        )
+    )
+    return format_tf_content(
+        "\n".join(
+            [
+                *([] if config.skip_python else [locals_def(python_module)]),
+                *([] if config.skip_python else [data_external(python_module, config)]),
+                "",
+                resource_hcl,
+                "",
+            ]
+        )
+    )

atlas_init/tf_ext/gen_resource_output.py

@@ -0,0 +1,71 @@
+from dataclasses import fields
+from typing import Iterable
+from atlas_init.tf_ext.models_module import ContainerType, ResourceTypePythonModule, ModuleGenConfig, ResourceAbs
+
+
+def as_output(resource_type: str, field_name: str, output_name: str) -> str:
+    return _as_output(output_name, f"{resource_type}.this.{field_name}")
+
+
+def _as_output(name: str, value: str) -> str:
+    return f"""\
+output "{name}" {{
+    value = {value}
+}}
+"""
+
+
+def as_nested_output(
+    resource_type: str,
+    parent_cls: type[ResourceAbs],
+    nested_types: dict[str, ContainerType[ResourceAbs]],
+    config: ModuleGenConfig,
+) -> Iterable[str]:
+    resource_id = f"{resource_type}.this"
+    for field_name, container_type in nested_types.items():
+        if container_type.is_any:
+            continue
+        computed_nested_fields = [
+            nested_field.name
+            for nested_field in fields(container_type.type)
+            if ResourceAbs.is_computed_only(nested_field.name, container_type.type)
+        ]
+        if container_type.is_list:
+            for computed_field_name in computed_nested_fields:
+                if container_type.is_optional and not ResourceAbs.is_required(field_name, parent_cls):
+                    yield _as_output(
+                        config.output_name(resource_type, field_name, computed_field_name),
+                        f"{resource_id}.{field_name} == null ? null : {resource_id}.{field_name}[*].{computed_field_name}",
+                    )
+                else:
+                    yield _as_output(
+                        config.output_name(resource_type, field_name, computed_field_name),
+                        f"{resource_id}.{field_name}[*].{computed_field_name}",
+                    )
+        elif container_type.is_set:
+            continue  # block type "limits" is represented by a set of objects, and set elements do not have addressable keys. To find elements matching specific criteria, use a "for" expression with an "if" clause.
+        elif container_type.is_dict or container_type.is_set:
+            raise NotImplementedError("Dict and set container types not supported yet")
+        else:
+            for computed_field_name in computed_nested_fields:
+                if container_type.is_optional and not ResourceAbs.is_required(field_name, parent_cls):
+                    yield _as_output(
+                        config.output_name(resource_type, field_name, computed_field_name),
+                        f"{resource_id}.{field_name} == null ? null : {resource_id}.{field_name}.{computed_field_name}",
+                    )
+                else:
+                    yield _as_output(
+                        config.output_name(resource_type, field_name, computed_field_name),
+                        f"{resource_id}.{field_name}.{computed_field_name}",
+                    )
+
+
+def generate_resource_output(py_module: ResourceTypePythonModule, config: ModuleGenConfig) -> str:
+    nested_types = dict(py_module.nested_field_types)
+    base_resource = py_module.resource
+    assert base_resource is not None, f"Resource {py_module.resource_type} has no base resource"
+    computed_field_names = [name for name in py_module.base_field_names_computed if name not in nested_types]
+    return "\n".join(
+        as_output(py_module.resource_type, field_name, config.output_name(py_module.resource_type, field_name))
+        for field_name in computed_field_names
+    ) + "\n".join(as_nested_output(py_module.resource_type, base_resource, nested_types, config))