atlas-init 0.3.0__py3-none-any.whl → 0.3.2__py3-none-any.whl

atlas_init/__init__.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-VERSION = "0.3.0"
+VERSION = "0.3.2"
 
 
 def running_in_repo() -> bool:

atlas_init/cli_tf/app.py

@@ -30,7 +30,6 @@ from atlas_init.cli_tf.go_test_summary import (
 )
 from atlas_init.cli_tf.mock_tf_log import mock_tf_log_cmd
 from atlas_init.cli_tf.schema import (
-    download_admin_api,
     dump_generator_config,
     parse_py_terraform_schema,
     update_provider_code_spec,
@@ -42,6 +41,7 @@ from atlas_init.cli_tf.schema_v2 import (
 )
 from atlas_init.cli_tf.schema_v2_api_parsing import add_api_spec_info
 from atlas_init.cli_tf.schema_v2_sdk import generate_model_go, parse_sdk_model
+from atlas_init.repos.go_sdk import download_admin_api
 from atlas_init.repos.path import Repo, current_repo_path
 from atlas_init.settings.env_vars import init_settings
 from atlas_init.settings.interactive import confirm

atlas_init/cli_tf/debug_logs.py

@@ -98,12 +98,12 @@ class SDKRoundtrip(Entity):
     @property
     def version(self) -> str:
         content_type = self.response.headers.get("Content-Type", "v1")
-        try:
+        content_type_req = self.request.headers.get("Accept", "v1")
+        with suppress(ValueError):
             return extract_version(content_type)
-        except ValueError:
-            logger.warning(f"failed to extract version from response header ({content_type}), trying request")
-        content_type = self.request.headers.get("Accept", "v1")
-        return extract_version(content_type)
+        with suppress(ValueError):
+            return extract_version(content_type_req)
+        raise ValueError(f"Could not extract version from req/resp: {content_type} or {content_type_req}")
 
     @model_validator(mode="after")
     def ensure_match(self) -> Self:
@@ -159,7 +159,7 @@ def parse_http_requests(logs: str) -> list[SDKRoundtrip]:
     Can say that expected payload is either a list or a dict and if it ends with an identifier it is higher chance for a dict
     """
     test_name = parse_test_name(logs)
-    logger.info(f"Finding http requests for test name: {test_name}")
+    logger.info(f"Finding http requests for test name: '{test_name}'")
     requests, responses = parse_raw_req_responses(logs)
     tf_step_starts = [i for i, line in enumerate(logs.splitlines()) if MARKER_START_STEP in line]
     used_responses: set[int] = set()

atlas_init/cli_tf/debug_logs_test_data.py

@@ -31,14 +31,7 @@ class RequestInfo(Entity):
 
     @property
     def id(self):
-        return "__".join(  # noqa: FLY002
-            [
-                self.method,
-                self.path,
-                self.version,
-                self.text,
-            ]  # need to include text to differentiate between requests
-        )
+        return "__".join(part for part in (self.method, self.path, self.version, self.text) if part)
 
 
 class StepRequests(Entity):
@@ -150,9 +143,9 @@ class MockRequestData(Entity):
     def replace_text_variables(self):
         for step in self.steps:
             for request in step.all_requests:
-                request.text = normalize_text(request.text, self.variables)
+                request.text = normalize_text(request.text, self.variables, expect_json=True)
                 for response in request.responses:
-                    response.text = normalize_text(response.text, self.variables)
+                    response.text = normalize_text(response.text, self.variables, expect_json=True)
 
     def prune_duplicate_responses(self):
         for step in self.steps:
@@ -209,10 +202,10 @@ def find_normalized_path(path: str, api_spec_paths: list[ApiSpecPath]) -> ApiSpe
     raise ValueError(f"Could not find path: {path}")
 
 
-def normalize_text(text: str, variables: dict[str, str]) -> str:
+def normalize_text(text: str, variables: dict[str, str], *, expect_json: bool = False) -> str:
     for var, value in variables.items():
         text = text.replace(value, f"{{{var}}}")
-    if not text:
+    if not text or not expect_json:
         return text
     try:
         parsed_text = json.loads(text)
@@ -267,8 +260,8 @@ def create_mock_data(
         for modifier in modifiers:
             if modifier.match(rt, normalized_path):
                 modifier.modification(rt)
-        normalized_text = normalize_text(rt.request.text, mock_data.variables)
-        normalized_response_text = normalize_text(rt.response.text, mock_data.variables)
+        normalized_text = normalize_text(rt.request.text, mock_data.variables, expect_json=True)
+        normalized_response_text = normalize_text(rt.response.text, mock_data.variables, expect_json=True)
         mock_data.add_roundtrip(rt, normalized_path, normalized_text, normalized_response_text, is_diff(rt))
     mock_data.replace_text_variables()
     if prune_duplicates:

atlas_init/cli_tf/mock_tf_log.py

@@ -1,10 +1,14 @@
 import json
 import logging
+import time
+from collections.abc import Callable
+from io import StringIO
 from pathlib import Path
 from typing import Self
 
 import typer
-from model_lib import Entity, dump
+import yaml
+from model_lib import Entity
 from pydantic import Field, model_validator
 from zero_3rdparty import file_utils
 
@@ -14,8 +18,17 @@ from atlas_init.cli_tf.debug_logs import (
     parse_http_requests,
     parse_test_name,
 )
-from atlas_init.cli_tf.debug_logs_test_data import create_mock_data, default_is_diff
-from atlas_init.repos.go_sdk import parse_api_spec_paths
+from atlas_init.cli_tf.debug_logs_test_data import (
+    RTModifier,
+    create_mock_data,
+    default_is_diff,
+)
+from atlas_init.repos.go_sdk import (
+    api_spec_path_transformed,
+    download_admin_api,
+    parse_api_spec_paths,
+)
+from atlas_init.settings.path import DEFAULT_DOWNLOADS_DIR
 
 logger = logging.getLogger(__name__)
 
@@ -23,18 +36,20 @@ logger = logging.getLogger(__name__)
 class MockTFLog(Entity):
     log_path: Path
     output_dir: Path
-    sdk_path: Path
+    admin_api_path: Path
     diff_skip_suffixes: list[str] = Field(default_factory=list)
     keep_duplicates: bool = False
+    modifiers: list[RTModifier] = Field(default_factory=list)
+    log_diff_roundtrips: bool = False
 
     @model_validator(mode="after")
     def ensure_paths_exist(self) -> Self:
         if not self.log_path.exists():
-            raise ValueError(f"log_path: {self.log_path} doesn't exist")
-        if not self.sdk_path.exists():
-            raise ValueError(f"sdk_path: {self.sdk_path} doesn't exist")
+            raise ValueError(f"log_path: '{self.log_path}' doesn't exist")
+        if not self.admin_api_path.exists():
+            raise ValueError(f"admin_api_path: '{self.admin_api_path}' doesn't exist")
         if not self.output_dir.exists():
-            raise ValueError(f"output_dir: {self.output_dir} doesn't exist")
+            raise ValueError(f"output_dir: '{self.output_dir}' doesn't exist")
         assert self.output_dir.name == "testdata", "output_path should be a directory named testdata"
         return self
 
@@ -42,44 +57,114 @@ class MockTFLog(Entity):
         return default_is_diff(rt) and not any(rt.request.path.endswith(suffix) for suffix in self.diff_skip_suffixes)
 
 
-def mock_tf_log(req: MockTFLog) -> None:
+def mock_tf_log(req: MockTFLog) -> Path:
     log_file_text = req.log_path.read_text()
     test_name = parse_test_name(log_file_text)
     roundtrips = parse_http_requests(log_file_text)
-    api_spec_paths = parse_api_spec_paths(req.sdk_path)
+    logger.info(f"Found #{len(roundtrips)} roundtrips")
+    if req.log_diff_roundtrips:
+        log_diff_roundtrips(roundtrips, req.differ)
+    api_spec_paths = parse_api_spec_paths(req.admin_api_path)
     data = create_mock_data(
         roundtrips,
         api_spec_paths,
         is_diff=req.differ,
         prune_duplicates=not req.keep_duplicates,
+        modifiers=req.modifiers,
     )
     # avoid anchors
-    data_yaml = dump(json.loads(dump(data, "json")), "yaml")
+    data_json = data.model_dump_json(exclude_none=True)
+    data_parsed = json.loads(data_json)
+    s = StringIO()
+    yaml.safe_dump(
+        data_parsed,
+        s,
+        default_flow_style=False,
+        width=100_000,
+        allow_unicode=True,
+        sort_keys=False,
+    )
+    data_yaml = s.getvalue()
+    test_name = test_name.replace("TestAcc", "TestMock")
     output_path = req.output_dir / f"{test_name}.yaml"
+    logger.info(f"Variables found {data.variables}")
     logger.info(f"Writing to {output_path}")
     file_utils.ensure_parents_write_text(output_path, data_yaml)
+    return output_path
 
 
 def mock_tf_log_cmd(
     log_path: str = typer.Argument(..., help="the path to the log file generated with TF_LOG_PATH"),
-    sdk_repo_path_str: str = option_sdk_repo_path,
     output_testdir: str = typer.Option(
         "",
         "-o",
         "--output-testdir",
-        help="the path to the output test directory, for example: internal/service/advancedclustertpf/testdata/",
+        help="the path to the output test directory, for example: internal/service/advancedclustertpf/testdata/, uses $(cwd)/testdata by default",
+    ),
+    sdk_repo_path_str: str = option_sdk_repo_path,
+    sdk_branch: str = typer.Option("main", "-b", "--branch", help="the branch for downloading openapi spec"),
+    admin_api_path: str = typer.Option(
+        "", "-a", "--admin-api-path", help="the path to store/download the openapi spec"
     ),
     diff_skip_suffixes: list[str] = typer.Option(..., "-s", "--skip-suffixes", default_factory=list),
     keep_duplicates: bool = typer.Option(False, "-keep", "--keep-duplicates", help="keep duplicate requests"),
+    log_diff_roundtrips: bool = typer.Option(
+        False, "-l", "--log-diff-roundtrips", help="print out the roundtrips used in diffs"
+    ),
 ):
     cwd = Path.cwd()
-    default_sdk_path = cwd.parent / "atlas-sdk-go"
-    default_testdir = cwd.parent / "testdata"
+    default_testdir = cwd / "testdata"
+    resolved_admin_api_path = resolve_admin_api_path(sdk_repo_path_str, sdk_branch, admin_api_path)
     event_in = MockTFLog(
         log_path=Path(log_path),
         output_dir=Path(output_testdir) if output_testdir else default_testdir,
-        sdk_path=Path(sdk_repo_path_str) if sdk_repo_path_str else default_sdk_path,
+        admin_api_path=resolved_admin_api_path,
         diff_skip_suffixes=diff_skip_suffixes,
         keep_duplicates=keep_duplicates,
+        log_diff_roundtrips=log_diff_roundtrips,
     )
     mock_tf_log(event_in)
+
+
+def is_cache_up_to_date(cache_path: Path, cache_ttl: int) -> bool:
+    if cache_path.exists():
+        modified_ts = file_utils.file_modified_time(cache_path)
+        if modified_ts > time.time() - cache_ttl:
+            logger.info(f"using cached admin api: {cache_path} downloaded {time.time()-modified_ts:.0f}s ago")
+            return True
+    return False
+
+
+def resolve_admin_api_path(sdk_repo_path_str: str, sdk_branch: str, admin_api_path: str) -> Path:
+    if admin_api_path:
+        resolved_admin_api_path = Path(admin_api_path)
+        if not resolved_admin_api_path.exists():
+            download_admin_api(resolved_admin_api_path, sdk_branch)
+    elif sdk_repo_path_str:
+        sdk_repo_path = Path(sdk_repo_path_str)
+        assert sdk_repo_path.exists(), f"not found sdk_repo_path={sdk_repo_path}"
+        resolved_admin_api_path = api_spec_path_transformed(sdk_repo_path)
+    else:
+        resolved_admin_api_path = DEFAULT_DOWNLOADS_DIR / "atlas-api-transformed.yaml"
+        if not is_cache_up_to_date(resolved_admin_api_path, 3600):
+            download_admin_api(resolved_admin_api_path, sdk_branch)
+    assert resolved_admin_api_path.exists(), f"unable to resolve admin_api_path={resolved_admin_api_path}"
+    assert resolved_admin_api_path.is_file(), f"not a file admin_api_path={resolved_admin_api_path}"
+    return resolved_admin_api_path
+
+
+def log_diff_roundtrips(roundtrips: list[SDKRoundtrip], differ: Callable[[SDKRoundtrip], bool] | None = None):
+    differ = differ or default_is_diff
+    diff_count = 0
+    step_nr = 0
+    for rt in roundtrips:
+        if not differ(rt):
+            continue
+        if rt.step_number != step_nr:
+            logger.info(f"{'-' * 80}\nStep {rt.step_number}")
+            step_nr = rt.step_number
+        diff_count += 1
+        logger.info(
+            f"\n{rt.request.method} {rt.request.path}\n{rt.request.text}\n{rt.response.status}-{rt.response.status_text}\n{rt.response.text}"
+        )
+    logger.info(f"Diffable requests: {diff_count}")

atlas_init/cli_tf/schema.py

@@ -5,7 +5,6 @@ from pathlib import Path
 from typing import Annotated, Literal, NamedTuple
 
 import pydantic
-import requests
 from model_lib import Entity, dump, field_names, parse_model
 from zero_3rdparty import dict_nested
 from zero_3rdparty.enum_utils import StrEnum
@@ -66,14 +65,19 @@ class SkipValidators(Entity):
     type: Literal["skip_validators"] = "skip_validators"
 
 
-Extension = Annotated[IgnoreNested | RenameAttribute | ChangeAttributeType | SkipValidators, pydantic.Field("type")]
+Extension = Annotated[
+    IgnoreNested | RenameAttribute | ChangeAttributeType | SkipValidators,
+    pydantic.Field("type"),
+]
 
 
 class TFResource(Entity):
     model_config = pydantic.ConfigDict(extra="allow")
     name: str
     extensions: list[Extension] = pydantic.Field(default_factory=list)
-    provider_spec_attributes: list[ProviderSpecAttribute] = pydantic.Field(default_factory=list)
+    provider_spec_attributes: list[ProviderSpecAttribute] = pydantic.Field(
+        default_factory=list
+    )
 
     def dump_generator_config(self) -> dict:
         names = field_names(self)
@@ -129,7 +133,9 @@ class ProviderCodeSpec(Entity):
             raise ValueError(f"{self.root_name(name, is_datasource)} not found!")
         return root_value
 
-    def schema_attributes(self, name: str, is_datasource: bool = False) -> list:  # noqa: FBT002
+    def schema_attributes(
+        self, name: str, is_datasource: bool = False
+    ) -> list:
         root_dict = self.root_dict(name, is_datasource)
         return root_dict["schema"]["attributes"]
 
@@ -139,16 +145,26 @@ class ProviderCodeSpec(Entity):
     def root_name(self, name: str, is_datasource: bool):
         return f"{self._type_name(is_datasource)}.{name}"
 
-    def attribute_names(self, name: str, is_datasource: bool = False) -> list[str]:  # noqa: FBT002
-        return [a["name"] for a in self.schema_attributes(name, is_datasource=is_datasource)]
+    def attribute_names(
+        self, name: str, is_datasource: bool = False
+    ) -> list[str]:
+        return [
+            a["name"] for a in self.schema_attributes(name, is_datasource=is_datasource)
+        ]
 
-    def iter_all_attributes(self, name: str, is_datasource: bool = False) -> Iterable[AttributeTuple]:  # noqa: FBT002
+    def iter_all_attributes(
+        self, name: str, is_datasource: bool = False
+    ) -> Iterable[AttributeTuple]:
         for attribute in self.schema_attributes(name=name, is_datasource=is_datasource):
             yield AttributeTuple(attribute["name"], "", attribute)
         yield from self.iter_nested_attributes(name, is_datasource=is_datasource)
 
-    def iter_nested_attributes(self, name: str, is_datasource: bool = False) -> Iterable[AttributeTuple]:  # noqa: FBT002
-        for i, attribute in enumerate(self.schema_attributes(name=name, is_datasource=is_datasource)):
+    def iter_nested_attributes(
+        self, name: str, is_datasource: bool = False
+    ) -> Iterable[AttributeTuple]:
+        for i, attribute in enumerate(
+            self.schema_attributes(name=name, is_datasource=is_datasource)
+        ):
             for path, attr_dict in dict_nested.iter_nested_key_values(
                 attribute, type_filter=dict, include_list_indexes=True
             ):
@@ -156,32 +172,53 @@ class ProviderCodeSpec(Entity):
                 if name := attr_dict.get("name", ""):
                     yield AttributeTuple(name, full_path, attr_dict)
 
-    def remove_nested_attribute(self, name: str, path: str, is_datasource: bool = False) -> None:  # noqa: FBT002
+    def remove_nested_attribute(
+        self, name: str, path: str, is_datasource: bool = False
+    ) -> None:
         root_name = self.root_name(name, is_datasource)
         logger.info(f"will remove attribute from {root_name} with path: {path}")
         root_attributes = self.root_dict(name, is_datasource)
         full_path = f"schema.attributes.{path}"
         popped = dict_nested.pop_nested(root_attributes, full_path, "")
         if popped == "":
-            raise ValueError(f"failed to remove attribute from resource {name} with path: {full_path}")
-        assert isinstance(popped, dict), f"expected removed attribute to be a dict, got: {popped}"
+            raise ValueError(
+                f"failed to remove attribute from resource {name} with path: {full_path}"
+            )
+        assert isinstance(
+            popped, dict
+        ), f"expected removed attribute to be a dict, got: {popped}"
         logger.info(f"removal ok, attribute_name: '{root_name}.{popped.get('name')}'")
 
-    def read_attribute(self, name: str, path: str, *, is_datasource: bool = False) -> dict:
+    def read_attribute(
+        self, name: str, path: str, *, is_datasource: bool = False
+    ) -> dict:
         if "." not in path:
-            attribute_dict = next((a for a in self.schema_attributes(name, is_datasource) if a["name"] == path), None)
+            attribute_dict = next(
+                (
+                    a
+                    for a in self.schema_attributes(name, is_datasource)
+                    if a["name"] == path
+                ),
+                None,
+            )
         else:
             root_dict = self.root_dict(name, is_datasource)
-            attribute_dict = dict_nested.read_nested_or_none(root_dict, f"schema.attributes.{path}")
+            attribute_dict = dict_nested.read_nested_or_none(
+                root_dict, f"schema.attributes.{path}"
+            )
         if attribute_dict is None:
-            raise ValueError(f"attribute {path} not found in {self.root_name(name, is_datasource)}")
+            raise ValueError(
+                f"attribute {path} not found in {self.root_name(name, is_datasource)}"
+            )
         assert isinstance(
             attribute_dict, dict
         ), f"expected attribute to be a dict, got: {attribute_dict} @ {path} for resource={name}"
         return attribute_dict
 
 
-def update_provider_code_spec(schema: PyTerraformSchema, provider_code_spec_path: Path) -> str:
+def update_provider_code_spec(
+    schema: PyTerraformSchema, provider_code_spec_path: Path
+) -> str:
     spec = parse_model(provider_code_spec_path, t=ProviderCodeSpec)
     for resource in schema.resources:
         resource_name = resource.name
@@ -192,41 +229,67 @@ def update_provider_code_spec(schema: PyTerraformSchema, provider_code_spec_path
     for data_source in schema.data_sources:
         data_source_name = data_source.name
         if extra_spec_attributes := data_source.provider_spec_attributes:
-            add_explicit_attributes(spec, data_source_name, extra_spec_attributes, is_datasource=True)
+            add_explicit_attributes(
+                spec, data_source_name, extra_spec_attributes, is_datasource=True
+            )
         for extension in data_source.extensions:
             apply_extension(extension, spec, data_source_name, is_datasource=True)
     return dump(spec, "json")
 
 
 def add_explicit_attributes(
-    spec: ProviderCodeSpec, name: str, extra_spec_attributes: list[ProviderSpecAttribute], *, is_datasource=False
+    spec: ProviderCodeSpec,
+    name: str,
+    extra_spec_attributes: list[ProviderSpecAttribute],
+    *,
+    is_datasource=False,
 ):
     resource_attributes = spec.schema_attributes(name, is_datasource=is_datasource)
     existing_names = spec.attribute_names(name, is_datasource=is_datasource)
     new_names = [extra.name for extra in extra_spec_attributes]
     if both := set(existing_names) & set(new_names):
         raise ValueError(f"resource: {name}, has already: {both} attributes")
-    resource_attributes.extend(extra.dump_provider_code_spec() for extra in extra_spec_attributes)
+    resource_attributes.extend(
+        extra.dump_provider_code_spec() for extra in extra_spec_attributes
+    )
 
 
 @singledispatch
-def apply_extension(extension: object, spec: ProviderCodeSpec, resource_name: str, *, is_datasource: bool = False):  # noqa: ARG001
+def apply_extension(
+    extension: object,
+    spec: ProviderCodeSpec,
+    resource_name: str,
+    *,
+    is_datasource: bool = False,
+):
     raise NotImplementedError(f"unsupported extension: {extension!r}")
 
 
 @apply_extension.register  # type: ignore
-def _ignore_nested(extension: IgnoreNested, spec: ProviderCodeSpec, resource_name: str, *, is_datasource: bool = False):
+def _ignore_nested(
+    extension: IgnoreNested,
+    spec: ProviderCodeSpec,
+    resource_name: str,
+    *,
+    is_datasource: bool = False,
+):
     if extension.use_wildcard:
         name_to_remove = extension.path.removeprefix("*.")
-        assert "*" not in name_to_remove, f"only prefix *. is allowed for wildcard in path {extension.path}"
+        assert (
+            "*" not in name_to_remove
+        ), f"only prefix *. is allowed for wildcard in path {extension.path}"
         found_paths = [
             path
-            for name, path, attribute_dict in spec.iter_nested_attributes(resource_name, is_datasource=is_datasource)
+            for name, path, attribute_dict in spec.iter_nested_attributes(
+                resource_name, is_datasource=is_datasource
+            )
             if name == name_to_remove
         ]
         while found_paths:
             next_to_remove = found_paths.pop()
-            spec.remove_nested_attribute(resource_name, next_to_remove, is_datasource=is_datasource)
+            spec.remove_nested_attribute(
+                resource_name, next_to_remove, is_datasource=is_datasource
+            )
             found_paths = [
                 path
                 for name, path, attribute_dict in spec.iter_nested_attributes(
@@ -241,9 +304,15 @@ def _ignore_nested(extension: IgnoreNested, spec: ProviderCodeSpec, resource_nam
 
 @apply_extension.register  # type: ignore
 def _rename_attribute(
-    extension: RenameAttribute, spec: ProviderCodeSpec, resource_name: str, *, is_datasource: bool = False
+    extension: RenameAttribute,
+    spec: ProviderCodeSpec,
+    resource_name: str,
+    *,
+    is_datasource: bool = False,
 ):
-    for attribute_dict in spec.schema_attributes(resource_name, is_datasource=is_datasource):
+    for attribute_dict in spec.schema_attributes(
+        resource_name, is_datasource=is_datasource
+    ):
         if attribute_dict.get("name") == extension.from_name:
             logger.info(
                 f"renaming attribute for {spec.root_name(resource_name, is_datasource)}: {extension.from_name} -> {extension.to_name}"
@@ -253,9 +322,15 @@ def _rename_attribute(
 
 @apply_extension.register  # type: ignore
 def _change_attribute_type(
-    extension: ChangeAttributeType, spec: ProviderCodeSpec, resource_name: str, *, is_datasource: bool = False
+    extension: ChangeAttributeType,
+    spec: ProviderCodeSpec,
+    resource_name: str,
+    *,
+    is_datasource: bool = False,
 ):
-    attribute_dict = spec.read_attribute(resource_name, extension.path, is_datasource=is_datasource)
+    attribute_dict = spec.read_attribute(
+        resource_name, extension.path, is_datasource=is_datasource
+    )
     old_value = extension.read_value(attribute_dict)
     if old_value == extension.new_value:
         logger.info(
@@ -270,31 +345,25 @@ def _change_attribute_type(
 
 
 @apply_extension.register  # type: ignore
-def _skip_validators(_: SkipValidators, spec: ProviderCodeSpec, resource_name: str, *, is_datasource: bool = False):
-    for attr_tuple in spec.iter_all_attributes(resource_name, is_datasource=is_datasource):
+def _skip_validators(
+    _: SkipValidators,
+    spec: ProviderCodeSpec,
+    resource_name: str,
+    *,
+    is_datasource: bool = False,
+):
+    for attr_tuple in spec.iter_all_attributes(
+        resource_name, is_datasource=is_datasource
+    ):
         attribute_dict = attr_tuple.attribute_dict
         paths_to_pop = [
             f"{path}.validators"
-            for path, nested_dict in dict_nested.iter_nested_key_values(attribute_dict, type_filter=dict)
+            for path, nested_dict in dict_nested.iter_nested_key_values(
+                attribute_dict, type_filter=dict
+            )
             if "validators" in nested_dict
         ]
         if paths_to_pop:
             logger.info(f"popping validators from '{attr_tuple.attribute_path}'")
         for path in paths_to_pop:
             dict_nested.pop_nested(attribute_dict, path)
-
-
-# reusing url from terraform-provider-mongodbatlas/scripts/schema-scaffold.sh
-ADMIN_API_URL = "https://raw.githubusercontent.com/mongodb/atlas-sdk-go/main/openapi/atlas-api-transformed.yaml"
-
-
-def admin_api_url(branch: str) -> str:
-    return ADMIN_API_URL.replace("/main/", f"/{branch}/")
-
-
-def download_admin_api(dest: Path, branch: str = "main") -> None:
-    url = admin_api_url(branch)
-    logger.info(f"downloading admin api to {dest} from {url}")
-    response = requests.get(url, timeout=10)
-    response.raise_for_status()
-    dest.write_bytes(response.content)

atlas_init/repos/go_sdk.py

@@ -1,9 +1,11 @@
 from collections import defaultdict
 from pathlib import Path
 
+import requests
 from model_lib import parse_model
 
 from atlas_init.cli_tf.debug_logs_test_data import ApiSpecPath
+from atlas_init.cli_tf.schema import logger
 from atlas_init.cli_tf.schema_v2_api_parsing import OpenapiSchema
 
 
@@ -15,11 +17,31 @@ def go_sdk_breaking_changes(repo_path: Path, go_sdk_rel_path: str = "../atlas-sd
     return breaking_changes_dir
 
 
-def parse_api_spec_paths(sdk_repo_path: Path) -> dict[str, list[ApiSpecPath]]:
-    api_spec_path = sdk_repo_path / "openapi/atlas-api-transformed.yaml"
+def api_spec_path_transformed(sdk_repo_path: Path) -> Path:
+    return sdk_repo_path / "openapi/atlas-api-transformed.yaml"
+
+
+def parse_api_spec_paths(api_spec_path: Path) -> dict[str, list[ApiSpecPath]]:
     model = parse_model(api_spec_path, t=OpenapiSchema)
     paths: dict[str, list[ApiSpecPath]] = defaultdict(list)
     for path, path_dict in model.paths.items():
         for method in path_dict:
             paths[method.upper()].append(ApiSpecPath(path=path))
     return paths
+
+
+# reusing url from terraform-provider-mongodbatlas/scripts/schema-scaffold.sh
+ADMIN_API_URL = "https://raw.githubusercontent.com/mongodb/atlas-sdk-go/main/openapi/atlas-api-transformed.yaml"
+
+
+def admin_api_url(branch: str) -> str:
+    return ADMIN_API_URL.replace("/main/", f"/{branch}/")
+
+
+def download_admin_api(dest: Path, branch: str = "main") -> None:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    url = admin_api_url(branch)
+    logger.info(f"downloading admin api to {dest} from {url}")
+    response = requests.get(url, timeout=10)
+    response.raise_for_status()
+    dest.write_bytes(response.content)

atlas_init/settings/path.py

@@ -28,6 +28,7 @@ DEFAULT_CONFIG_PATH = ROOT_PATH / "atlas_init.yaml"
 DEFAULT_SCHEMA_CONFIG_PATH = ROOT_PATH / "terraform.yaml"
 DEFAULT_GITHUB_CI_RUN_LOGS = ROOT_PATH / "github_ci_run_logs"
 DEFAULT_GITHUB_SUMMARY_DIR = ROOT_PATH / "github_ci_summary"
+DEFAULT_DOWNLOADS_DIR = ROOT_PATH / "downloads"
 
 
 def load_dotenv(env_path: Path) -> dict[str, str]:

atlas_init/settings/rich_utils.py

@@ -46,7 +46,7 @@ def hide_secrets(handler: logging.Handler, secrets_dict: dict[str, str]) -> None
 
 def configure_logging(log_level: str = "INFO") -> logging.Handler:
     _LogLevel(log_level=log_level)  # type: ignore
-    handler = RichHandler(rich_tracebacks=True)
+    handler = RichHandler(rich_tracebacks=False)
     logging.basicConfig(
         level=logging.getLevelName(log_level),
         format="%(message)s",

atlas_init/typer_app.py

@@ -59,4 +59,8 @@ def main(
     logger.info(f"running in repo: {running_in_repo()} python location:{sys.executable}")
     if not show_secrets:
         hide_secrets(log_handler, {**os.environ})
-    logger.info(f"in the app callback, log-level: {log_level}, command: {ctx.command}")
+    logger.info(f"in the app callback, log-level: {log_level}, command: {format_cmd(ctx)}")
+
+
+def format_cmd(ctx: typer.Context) -> str:
+    return f"'{ctx.info_name} {ctx.invoked_subcommand}'"

{atlas_init-0.3.0.dist-info → atlas_init-0.3.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: atlas-init
-Version: 0.3.0
+Version: 0.3.2
 Project-URL: Documentation, https://github.com/EspenAlbert/atlas-init#readme
 Project-URL: Issues, https://github.com/EspenAlbert/atlas-init/issues
 Project-URL: Source, https://github.com/EspenAlbert/atlas-init

{atlas_init-0.3.0.dist-info → atlas_init-0.3.2.dist-info}/RECORD

@@ -1,11 +1,11 @@
-atlas_init/__init__.py,sha256=wefdLatWmYi2mgkWhGxe2aEGq2miR5yH7tbuwRX6TAc,372
+atlas_init/__init__.py,sha256=5-YoPkhK1DsNmcQTFT5lp8zxJ26l6WRY5tMcoGBug6o,372
 atlas_init/__main__.py,sha256=dY1dWWvwxRZMmnOFla6RSfti-hMeLeKdoXP7SVYqMUc,52
 atlas_init/atlas_init.yaml,sha256=GMyJVhKKRc7WzEu7fafmWgeTsDaExTLv7QvXOmE_Brg,1907
 atlas_init/cli.py,sha256=IiOEC_Jry6vrSDH3_OvsU50F-_3iVIS4tV6-R7659fY,9642
 atlas_init/cli_args.py,sha256=tiwUYAE0JBSl9lHV6VJ41vFCU90ChBZ4mKvi-YoF_HY,541
 atlas_init/humps.py,sha256=l0ZXXuI34wwd9TskXhCjULfGbUyK-qNmiyC6_2ow6kU,7339
 atlas_init/terraform.yaml,sha256=qPrnbzBEP-JAQVkYadHsggRnDmshrOJyiv0ckyZCxwY,2734
-atlas_init/typer_app.py,sha256=nxwKwr2GNZ6Zd8uXfJV8mZucEHT0KvfRyF_qxJ3IpYQ,1962
+atlas_init/typer_app.py,sha256=zbvYUlZrF4TZEPEwpa33fVSLVKcxRamuXCgF1FCUhCU,2068
 atlas_init/cli_cfn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/cli_cfn/app.py,sha256=iMukpUDgsAgZh_U_APGZB3gmewOo-3vtFK0byJuDz9w,6649
 atlas_init/cli_cfn/aws.py,sha256=GbohR7uczSGwQjLEYozCmlxbeIHo1uwQIJMwsh7kF7M,17894
@@ -21,16 +21,16 @@ atlas_init/cli_helper/tf_runner.py,sha256=OYdC-Y6i-xRh8_LCudKdtP7CEYEO9e67nVhhol
 atlas_init/cli_root/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/cli_root/trigger.py,sha256=oEgqb_l25tyYgUaFHEuChcOCJA7k3mnRa4D-Myz-Igs,5789
 atlas_init/cli_tf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-atlas_init/cli_tf/app.py,sha256=-50lyNwG5eh8nVCN6mCcIB_b4xd5aJjhEoK-XVHD8XU,11359
+atlas_init/cli_tf/app.py,sha256=0Y5c-Pc9ibOz6kXvFlL-yhH_fx1nHLgBgK9OAVqjX9s,11390
 atlas_init/cli_tf/changelog.py,sha256=biWYKf1pZvXZ-jEgcZ5q9sY7nTGrL2PuI0h9mCILf_g,3181
-atlas_init/cli_tf/debug_logs.py,sha256=lnB5BpcEooVzGd2RLxbwAVQs0ZYXzRKy5sHa0hftHI8,8799
-atlas_init/cli_tf/debug_logs_test_data.py,sha256=CP1FbRvaFXOOr5m0u-Y6YAvWpqn6WzH-XoPYBJZN0bE,9594
+atlas_init/cli_tf/debug_logs.py,sha256=r9kRMzgZGP7SB0eK7hhbxETfI28yyHFtSpvSFe9NGC4,8839
+atlas_init/cli_tf/debug_logs_test_data.py,sha256=G4pnuWJ7PAQd3NXRKAtwAPC6Ne-PgpzaTZHQ9waqxZI,9565
 atlas_init/cli_tf/github_logs.py,sha256=VD7qhlXNuG21eTuJ5VI7rsflp5WHSodfngkRVgQlumw,8114
 atlas_init/cli_tf/go_test_run.py,sha256=ZoQSvIasmWauFxZJrWL0ObFX-P0k-D3c_ep3OnPY4zs,5842
 atlas_init/cli_tf/go_test_run_format.py,sha256=OUd6QPHDeTzbwVuh6MhP-xXgjOOGP9W_sCLJ8KylBTs,1201
 atlas_init/cli_tf/go_test_summary.py,sha256=agr4SITgxchjgOzRpScoTUk-iG38QDLkpnsMtTW9GTY,5382
-atlas_init/cli_tf/mock_tf_log.py,sha256=tefWI5mS5lwOWCUidI_p7LeTaoyYCQl951LFgLYGEy8,3157
-atlas_init/cli_tf/schema.py,sha256=q0RUHb-6ORbQ41u2bZXaWWQLB-xE8Q_O7U1AFq2Gfng,12398
+atlas_init/cli_tf/mock_tf_log.py,sha256=u_d6c-lVo3eDddpW3V5r3FJfvQvoyrC8NnOGS1bYpR8,6572
+atlas_init/cli_tf/schema.py,sha256=iwvb4wD2Wba0MMu7ooTNAIi1jHbpLiXGPOT51_o_YW8,12431
 atlas_init/cli_tf/schema_go_parser.py,sha256=PiRfFFVnkhltxcGFfOCgH53wwzIEynw2BXmSfaINLL8,8294
 atlas_init/cli_tf/schema_inspection.py,sha256=ujLvGfg3baByND4nRD0drZoI45STxo3VfYvim-PfVOc,1764
 atlas_init/cli_tf/schema_table.py,sha256=1i6urBFNVpyopmLbDkYhL3pceKc9NJBCphfVIbm-K6Y,5229
@@ -50,14 +50,14 @@ atlas_init/cloud/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,
 atlas_init/cloud/aws.py,sha256=97kkURWHFAKDIw4704aFmyoeAfQKL11IXMyaQbZUt80,2473
 atlas_init/repos/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/repos/cfn.py,sha256=rjyVVxRhWL65tdAbEHT72UReK2h99Bj6RA4O2pBO-bc,2466
-atlas_init/repos/go_sdk.py,sha256=nh3lw9iw4lDGdHnhC8KK0PZTDMUGKvCHTMTuEtIUKNg,1058
+atlas_init/repos/go_sdk.py,sha256=1OzM9DjHEAzAAuI9ygoRRuhUK2gqpOhXExXRqhqa0tg,1793
 atlas_init/repos/path.py,sha256=wrT8e01OBoAHj8iMrxqutgqWu-BHPe9-bEWtcZRu238,4187
 atlas_init/settings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/settings/config.py,sha256=HIytZom8RRvpLGy6u8CpZ83tmFXI6v1tO3iSiuo08kc,6259
 atlas_init/settings/env_vars.py,sha256=q8Hj2LPJIg-PK0fCjrEigoPwTGIEbqjLEZckwgnkG8s,9688
 atlas_init/settings/interactive.py,sha256=Xy1Z5WMAOSaJ-vQI_4xjAbSR92rWQgnffwVoDT27L68,340
-atlas_init/settings/path.py,sha256=FQ6-SVgc_KyxUjU9GWqq5YjViLDEEpojnApk8EwlS6U,2570
-atlas_init/settings/rich_utils.py,sha256=JyVD_RuZTBeZdbnXN7B3WsYGkROhmypgzAym-zD92B4,1703
+atlas_init/settings/path.py,sha256=KkXysu6-0AuSjsvYGknYGJX1hL2j1RD-Fpf8KsVYpkE,2618
+atlas_init/settings/rich_utils.py,sha256=5LgJUmc9wyJTsoS6xWKadrT0MoQREDaKvEOCuBLDXRg,1704
 atlas_init/tf/.terraform.lock.hcl,sha256=DIojR50rr4fyLShYiQ-UpRV8z6vuBjwGWdK60FODoyM,6876
 atlas_init/tf/always.tf,sha256=ij6QKI8Lg0140bFZwOyiYK5c-2p5e7AGZ1qKbYyv6Os,1359
 atlas_init/tf/main.tf,sha256=DH0C8y9RDEHnSAZvL-TjE5MQjxj5ALfgk5zVO88cpZw,3960
@@ -86,7 +86,7 @@ atlas_init/tf/modules/vpc_peering/vpc_peering.tf,sha256=hJ3KJdGbLpOQednUpVuiJ0Cq
 atlas_init/tf/modules/vpc_privatelink/atlas-privatelink.tf,sha256=FloaaX1MNDvoMZxBnEopeLKyfIlq6kaX2dmx8WWlXNU,1298
 atlas_init/tf/modules/vpc_privatelink/variables.tf,sha256=gktHCDYD4rz6CEpLg5aiXcFbugw4L5S2Fqc52QYdJyc,255
 atlas_init/tf/modules/vpc_privatelink/versions.tf,sha256=G0u5V_Hvvrkux_tqfOY05pA-GzSp_qILpfx1dZaTGDc,237
-atlas_init-0.3.0.dist-info/METADATA,sha256=Kgl-C8C71uv9NU9s6q1uOfhF2fcUIAmYfU5mdzR1vSE,5650
-atlas_init-0.3.0.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-atlas_init-0.3.0.dist-info/entry_points.txt,sha256=oSNFIEAS9nUZyyZ8Fc-0F0U5j-NErygy01LpJVSHapQ,57
-atlas_init-0.3.0.dist-info/RECORD,,
+atlas_init-0.3.2.dist-info/METADATA,sha256=foETVpu_FKi1rkGOJB158pmAftoNYFiG9VNr1kPBPvA,5650
+atlas_init-0.3.2.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+atlas_init-0.3.2.dist-info/entry_points.txt,sha256=oSNFIEAS9nUZyyZ8Fc-0F0U5j-NErygy01LpJVSHapQ,57
+atlas_init-0.3.2.dist-info/RECORD,,