atlas-init 0.4.5__py3-none-any.whl → 0.7.0__py3-none-any.whl

atlas_init/repos/go_sdk.py

@@ -2,11 +2,11 @@ from collections import defaultdict
 from pathlib import Path
 
 import requests
-from model_lib import parse_model
+from model_lib import Entity, parse_model
 
-from atlas_init.cli_tf.debug_logs_test_data import ApiSpecPath
+from atlas_init.cli_tf.debug_logs_test_data import ApiSpecPath, find_normalized_path
 from atlas_init.cli_tf.schema import logger
-from atlas_init.cli_tf.schema_v2_api_parsing import OpenapiSchema
+from atlas_init.cli_tf.openapi import OpenapiSchema
 
 
 def go_sdk_breaking_changes(repo_path: Path, go_sdk_rel_path: str = "../atlas-sdk-go") -> Path:
@@ -21,6 +21,15 @@ def api_spec_path_transformed(sdk_repo_path: Path) -> Path:
     return sdk_repo_path / "openapi/atlas-api-transformed.yaml"
 
 
+class ApiSpecPaths(Entity):
+    method_paths: dict[str, list[ApiSpecPath]]
+
+    def normalize_path(self, method: str, path: str) -> str:
+        if path.startswith("/api/atlas/v1.0"):
+            return ""
+        return find_normalized_path(path, self.method_paths[method]).path
+
+
 def parse_api_spec_paths(api_spec_path: Path) -> dict[str, list[ApiSpecPath]]:
     model = parse_model(api_spec_path, t=OpenapiSchema)
     paths: dict[str, list[ApiSpecPath]] = defaultdict(list)

atlas_init/repos/path.py

@@ -1,4 +1,9 @@
+from __future__ import annotations
+import logging
+import re
+from collections import defaultdict
 from collections.abc import Callable
+from dataclasses import dataclass
 from enum import StrEnum
 from functools import lru_cache
 from pathlib import Path
@@ -8,6 +13,7 @@ from git import Repo as _GitRepo
 
 from atlas_init.settings.path import current_dir, repo_path_rel_path
 
+logger = logging.getLogger(__name__)
 GH_OWNER_TERRAFORM_PROVIDER_MONGODBATLAS = "mongodb/terraform-provider-mongodbatlas"
 GH_OWNER_MONGODBATLAS_CLOUDFORMATION_RESOURCES = "mongodb/mongodbatlas-cloudformation-resources"
 _KNOWN_OWNER_PROJECTS = {
@@ -59,10 +65,10 @@ def is_resource_call(repo_path: Path) -> Callable[[Path], bool]:
 
 
 def resource_dir(repo_path: Path, full_path: Path) -> Path:
-    dir_name = resource_name(repo_path, full_path)
-    if not dir_name:
+    if dir_name := resource_name(repo_path, full_path):
+        return resource_root(repo_path) / dir_name
+    else:
         raise ValueError(f"no resource name for {full_path}")
-    return resource_root(repo_path) / dir_name
 
 
 class Repo(StrEnum):
@@ -116,10 +122,10 @@ def resource_name(repo_path: Path, full_path: Path) -> str:
     is_resource = is_resource_call(repo_path)
     if not root.exists():
         raise ValueError(f"no resource root found for {repo_path}")
-    for parent in [full_path, *full_path.parents]:
-        if parent.parent == root and is_resource(parent):
-            return parent.name
-    return ""
+    return next(
+        (parent.name for parent in [full_path, *full_path.parents] if parent.parent == root and is_resource(parent)),
+        "",
+    )
 
 
 def find_paths(assert_repo: Repo | None = None) -> ResourcePaths:
@@ -153,3 +159,100 @@ def find_go_mod_dir(repo_path: Path):
             return go_mod.parent
     msg = "go.mod not found or more than 1 level deep"
     raise ValueError(msg)
+
+
+def find_test_names(file: Path, prefix: str = "Test") -> list[str]:
+    test_names = []
+    with file.open("r") as f:
+        for line in f:
+            if line.startswith(f"func {prefix}"):
+                test_name = line.split("(")[0].strip().removeprefix("func ")
+                test_names.append(test_name)
+    return sorted(test_names)
+
+
+class MultipleResourceNames(ValueError):
+    def __init__(self, names: list[str]):
+        super().__init__(f"multiple resource names found: {names}")
+        self.names = names
+
+
+def find_tf_resource_name_in_test(path: Path, provider_prefix: str = "mongodbatlas_") -> str:
+    candidates: set[str] = {
+        match.group(1) for match in re.finditer(rf"=\s\"{provider_prefix}([a-zA-Z0-9_]+)\.?", path.read_text())
+    }
+    if len(candidates) > 1:
+        pkg_name = path.parent.name
+        for candidate in candidates:
+            if candidate.replace("_", "") == pkg_name:
+                return candidate
+        logger.warning(f"multiple resource names found in {path}: {candidates}")
+        raise MultipleResourceNames(sorted(candidates))
+    return candidates.pop() if candidates else ""
+
+
+def find_pkg_test_names(pkg_path: Path, prefix: str = "Test") -> list[str]:
+    test_names = []
+    for test_file in pkg_path.glob("*_test.go"):
+        test_names.extend(find_test_names(test_file, prefix))
+    return sorted(test_names)
+
+
+def terraform_resource_test_names(
+    repo_path: Path, prefix: str = "Test", package_path: str = "internal/service"
+) -> dict[str, list[str]]:
+    """find all test names in the given package path"""
+    pkg_path = terraform_package_path(repo_path, package_path)
+    resource_dirs, _ = find_resource_dirs(pkg_path)
+    test_names = defaultdict(list)
+    for name, pkg_dir in resource_dirs.items():
+        for test_file in pkg_dir.glob("*_test.go"):
+            test_names[name].extend(find_test_names(test_file, prefix))
+    return test_names
+
+
+def terraform_resources(repo_path: Path, package_path: str = "internal/service") -> list[TFResoure]:
+    pkg_path = terraform_package_path(repo_path, package_path)
+    resource_dirs, _ = find_resource_dirs(pkg_path)
+    resources = []
+    for name, pkg_dir in resource_dirs.items():
+        test_names = find_pkg_test_names(pkg_dir)
+        resources.append(TFResoure(name=name, package_rel_path=str(pkg_dir.relative_to(repo_path)), tests=test_names))
+    return resources
+
+
+def terraform_package_path(repo_path: Path, package_path: str = "internal/service"):
+    pkg_path = repo_path / package_path
+    if not pkg_path.exists():
+        raise ValueError(f"package path not found: {pkg_path}")
+    return pkg_path
+
+
+def find_resource_dirs(pkg_path: Path) -> tuple[dict[str, Path], list[Path]]:
+    resource_dirs: dict[str, Path] = {}
+    non_resource_dirs: list[Path] = []
+    for pkg_dir in pkg_path.iterdir():
+        if not pkg_dir.is_dir():
+            continue
+        if pkg_dir.name == "testdata":
+            continue
+        found = False
+        for test_file in pkg_dir.glob("*_test.go"):
+            try:
+                if name := find_tf_resource_name_in_test(test_file):
+                    resource_dirs[name] = pkg_dir
+                    found = True
+            except MultipleResourceNames as e:
+                for name in e.names:
+                    resource_dirs[name] = pkg_dir
+                found = True
+        if not found:
+            non_resource_dirs.append(pkg_dir)
+    return resource_dirs, non_resource_dirs
+
+
+@dataclass
+class TFResoure:
+    name: str
+    package_rel_path: str
+    tests: list[str]

atlas_init/sdk_ext/go.py

@@ -0,0 +1,102 @@
+from contextlib import suppress
+import logging
+from pathlib import Path
+from ask_shell import confirm, run_and_wait
+from model_lib import dump, parse_model
+import typer
+from zero_3rdparty.file_utils import clean_dir, copy, ensure_parents_write_text
+from atlas_init.cli_args import ParsedPaths, option_sdk_repo_path, option_mms_repo_path
+from atlas_init.cli_tf.openapi import OpenapiSchema
+
+_go_mod_line = "replace go.mongodb.org/atlas-sdk/v20250312005 v20250312005.0.0 => ../atlas-sdk-go"
+logger = logging.getLogger(__name__)
+
+
+def go(
+    mms_path_str: str = option_mms_repo_path,
+    sdk_repo_path_str: str = option_sdk_repo_path,
+    mms_branch: str = typer.Option("master", "--mms-branch", help="Branch to use for mms"),
+    skip_mms_openapi: bool = typer.Option(
+        False, "-smms", "--skip-mms-openapi", help="Skip mms openapi generation, use existing file instead"
+    ),
+):
+    paths = ParsedPaths.from_strings(sdk_repo_path_str=sdk_repo_path_str, mms_path=mms_path_str)
+    mms_path = paths.mms_repo_path
+    assert mms_path, "mms_path is required"
+    sdk_path = paths.sdk_repo_path
+    assert sdk_path, "sdk_path is required"
+    openapi_path = safe_openapi_path(mms_path) if skip_mms_openapi else generate_openapi_spec(mms_path, mms_branch)
+    openapi_path = transform_openapi(openapi_path, sdk_path / "openapi/openapi-mms.yaml")
+    generate_go_sdk(sdk_path, openapi_path)
+    confirm(f"Have you remembered to add to your go.mod file: {_go_mod_line}")
+
+
+def transform_openapi(old: Path, dest_path: Path) -> Path:
+    api_spec = parse_model(old, t=OpenapiSchema)
+    new_api_spec = api_spec.model_dump()
+    for path in api_spec.paths.keys():
+        for method_name, method in api_spec.methods_with_name(path):
+            responses = method.get("responses", {})
+            for code, multi_responses in responses.items():
+                with suppress(AlreadySingleVersion):
+                    new_api_spec["paths"][path][method_name]["responses"][code]["content"] = use_a_single_version(
+                        multi_responses, api_spec, path
+                    )
+            if request_body := method.get("requestBody", {}):
+                with suppress(AlreadySingleVersion):
+                    new_api_spec["paths"][path][method_name]["requestBody"]["content"] = use_a_single_version(
+                        request_body, api_spec, path
+                    )
+    dest_yaml = dump(new_api_spec, "yaml")
+    ensure_parents_write_text(dest_path, dest_yaml)
+    return dest_path
+
+
+class AlreadySingleVersion(Exception):
+    pass
+
+
+def use_a_single_version(multi_content: dict, api_spec: OpenapiSchema, path: str) -> dict[str, dict]:
+    if api_versions := api_spec._unpack_schema_versions(multi_content):
+        if len(api_versions) > 1:
+            latest_version = max(api_versions)
+            last_header = f"application/vnd.atlas.{latest_version}+json"
+            old_content = multi_content["content"]
+            assert last_header in old_content, f"failed to find {last_header} for {path} in {old_content.keys()}"
+            return {last_header: old_content[last_header]}
+    raise AlreadySingleVersion
+
+
+def generate_openapi_spec(mms_path: Path, mms_branch: str) -> Path:
+    run_and_wait(f"git stash && git checkout {mms_branch}", cwd=mms_path)
+    bazelisk_bin_run = run_and_wait("mise which bazelisk", cwd=mms_path)
+    bazelisk_bin = bazelisk_bin_run.stdout_one_line
+    assert Path(bazelisk_bin).exists(), f"not found {bazelisk_bin}"
+    openapi_run = run_and_wait(f"{bazelisk_bin} run //server:mms-openapi", cwd=mms_path, print_prefix="mms-openapi")
+    assert openapi_run.clean_complete, f"failed to run {openapi_run}"
+    return safe_openapi_path(mms_path)
+
+
+def safe_openapi_path(mms_path: Path) -> Path:
+    openapi_path = mms_path / "server/openapi/services/openapi-mms.json"
+    assert openapi_path.exists(), f"not found {openapi_path}"
+    return openapi_path
+
+
+def generate_go_sdk(repo_path: Path, openapi_path: Path) -> None:
+    SDK_FOLDER = repo_path / "admin"
+    clean_dir(SDK_FOLDER, recreate=True)
+    generate_script = repo_path / "tools/scripts/generate.sh"
+    assert generate_script.exists(), f"not found {generate_script}"
+    openapi_folder = repo_path / "openapi"
+    openapi_dest_path = openapi_folder / openapi_path.name
+    if openapi_path != openapi_dest_path:
+        copy(openapi_path, openapi_dest_path)
+    generate_env = {
+        "OPENAPI_FOLDER": str(openapi_folder),
+        "OPENAPI_FILE_NAME": openapi_path.name,
+        "SDK_FOLDER": str(SDK_FOLDER),
+    }
+    run_and_wait(f"{generate_script}", cwd=repo_path / "tools", env=generate_env, print_prefix="go sdk create")
+    mockery_script = repo_path / "tools/scripts/generate_mocks.sh"
+    run_and_wait(f"{mockery_script}", cwd=repo_path / "tools", print_prefix="go sdk mockery")

atlas_init/sdk_ext/typer_app.py

@@ -0,0 +1,18 @@
+from ask_shell import configure_logging
+from typer import Typer
+
+
+def typer_main():
+    from atlas_init.sdk_ext import go
+
+    app = Typer(
+        name="sdk-ext",
+        help="SDK extension commands for Atlas Init",
+    )
+    app.command(name="go")(go.go)
+    configure_logging(app)
+    app()
+
+
+if __name__ == "__main__":
+    typer_main()

atlas_init/settings/config.py

@@ -12,7 +12,7 @@ from typing import Any
 from model_lib import Entity, IgnoreFalsy
 from pydantic import Field, model_validator
 
-from atlas_init.repos.path import as_repo_alias, go_package_prefix, owner_project_name, package_glob
+from atlas_init.repos.path import as_repo_alias, find_test_names, go_package_prefix, owner_project_name, package_glob
 
 logger = logging.getLogger(__name__)
 
@@ -95,11 +95,8 @@ class TestSuite(IgnoreFalsy):
         for package in packages:
             pkg_name = f"{go_package_prefix(repo_path)}/{package}"
             for go_file in repo_path.glob(f"{package}/*.go"):
-                with go_file.open() as f:
-                    for line in f:
-                        if line.startswith(f"func {prefix}"):
-                            test_name = line.split("(")[0].strip().removeprefix("func ")
-                            names[pkg_name][test_name] = go_file.parent
+                for name in find_test_names(go_file, prefix):
+                    names[pkg_name][name] = go_file.parent
         return names
 
     def is_active(self, repo_alias: str, change_paths: Iterable[str]) -> bool:

atlas_init/settings/env_vars.py

@@ -9,7 +9,7 @@ from typing import Any, NamedTuple, TypeVar
 
 from model_lib import StaticSettings, parse_payload
 from pydantic import BaseModel, ValidationError, field_validator
-from zero_3rdparty import iter_utils
+from zero_3rdparty import iter_utils, str_utils
 
 from atlas_init.settings.config import (
     AtlasInitConfig,
@@ -54,6 +54,7 @@ class AtlasInitSettings(StaticSettings):
     atlas_init_tf_src_path: Path = DEFAULT_TF_SRC_PATH  # /tf directory of repo
     atlas_init_tf_schema_config_path: Path = DEFAULT_ATLAS_INIT_SCHEMA_CONFIG_PATH  # /terraform.yaml
     atlas_init_schema_out_path: Path | None = None  # override this for the generated schema
+    atlas_init_static_html_path: Path | None = None
 
     atlas_init_cfn_profile: str = ""
     atlas_init_cfn_region: str = ""
@@ -64,6 +65,9 @@ class AtlasInitSettings(StaticSettings):
 
     non_interactive: bool = False
 
+    mongo_database: str = "atlas_init"
+    mongo_url: str = "mongodb://user:pass@localhost:27017?retryWrites=true&w=majority&authSource=admin"
+
     @property
     def is_interactive(self) -> bool:
         return not self.non_interactive
@@ -133,6 +137,17 @@ class AtlasInitSettings(StaticSettings):
     def github_ci_summary_dir(self) -> Path:
         return self.cache_root / "github_ci_summary"
 
+    def github_ci_summary_path(self, summary_name: str) -> Path:
+        return self.github_ci_summary_dir / str_utils.ensure_suffix(summary_name, ".md")
+
+    def github_ci_summary_details_path(self, summary_name: str, test_name: str) -> Path:
+        return self.github_ci_summary_path(summary_name).parent / self.github_ci_summary_details_rel_path(
+            summary_name, test_name
+        )
+
+    def github_ci_summary_details_rel_path(self, summary_name: str, test_name: str) -> str:
+        return f"{summary_name.removesuffix('.md')}_details/{test_name}.md"
+
     @property
     def go_test_logs_dir(self) -> Path:
         return self.cache_root / "go_test_logs"
@@ -234,6 +249,7 @@ def find_missing_env_vars(required_env_vars: list[str], manual_env_vars: dict[st
 
 def init_settings(
     *settings_classes: type[BaseModel],
+    skip_ambiguous_check: bool = False,
 ) -> AtlasInitSettings:
     settings = AtlasInitSettings.from_env()
     profile_env_vars = settings.manual_env_vars
@@ -241,7 +257,7 @@ def init_settings(
     if vscode_env_vars.exists():
         profile_env_vars |= load_dotenv(vscode_env_vars)
     required_env_vars = collect_required_env_vars(list(settings_classes))
-    ambiguous = detect_ambiguous_env_vars(profile_env_vars)
+    ambiguous = [] if skip_ambiguous_check else detect_ambiguous_env_vars(profile_env_vars)
     missing_env_vars = find_missing_env_vars(required_env_vars, profile_env_vars)
 
     if ambiguous:

atlas_init/settings/env_vars_generated.py

@@ -63,3 +63,5 @@ class RealmSettings(_EnvVarsGenerated):
 
 class AtlasSettingsWithProject(AtlasSettings):
     MONGODB_ATLAS_PROJECT_ID: str
+    MONGODB_ATLAS_PROJECT_OWNER_ID: str = ""
+    MONGODB_ATLAS_USER_EMAIL: str = ""

atlas_init/settings/interactive2.py

@@ -0,0 +1,134 @@
+"""Inspired by: https://github.com/tmbo/questionary/blob/master/tests/utils.py"""
+
+from dataclasses import dataclass
+from typing import Callable, TypeVar
+
+from prompt_toolkit.input.defaults import create_pipe_input
+from prompt_toolkit.output import DummyOutput
+from questionary import Question, checkbox
+from questionary import confirm as _confirm
+from questionary import select as _select
+from questionary import text as _text
+
+T = TypeVar("T")
+TypedAsk = Callable[[Question, type[T]], T]
+
+_question_asker: TypedAsk = lambda q, _: q.ask()  # noqa: E731
+
+
+def confirm(prompt_text: str, *, default: bool | None = None) -> bool:
+    if default is None:
+        return _question_asker(_confirm(prompt_text), bool)
+    return _question_asker(_confirm(prompt_text, default=default), bool)
+
+
+def select_list_multiple(
+    prompt_text: str,
+    choices: list[str],
+    default: list[str] | None = None,
+) -> list[str]:
+    assert choices, "choices must not be empty"
+    default = default or []
+    return _question_asker(checkbox(prompt_text, choices=choices), list[str]) or default
+
+
+def text(
+    prompt_text: str,
+    default: str = "",
+) -> str:
+    return _question_asker(_text(prompt_text, default=default), str)
+
+
+T = TypeVar("T")
+
+
+def select_dict(
+    prompt_text: str,
+    choices: dict[str, T],
+    default: str | None = None,
+) -> T:
+    assert choices, "choices must not be empty"
+    selection = _question_asker(_select(prompt_text, default=default, choices=list(choices)), str)
+    return choices[selection]
+
+
+StrT = TypeVar("StrT", bound=str)
+
+
+def select_list(
+    prompt_text: str,
+    choices: list[StrT],
+    default: StrT | None = None,
+) -> StrT:
+    assert choices, "choices must not be empty"
+    return _question_asker(_select(prompt_text, default=default, choices=choices), str)
+
+
+class KeyInput:
+    DOWN = "\x1b[B"
+    UP = "\x1b[A"
+    LEFT = "\x1b[D"
+    RIGHT = "\x1b[C"
+    ENTER = "\r"
+    ESCAPE = "\x1b"
+    CONTROLC = "\x03"
+    CONTROLN = "\x0e"
+    CONTROLP = "\x10"
+    BACK = "\x7f"
+    SPACE = " "
+    TAB = "\x09"
+    ONE = "1"
+    TWO = "2"
+    THREE = "3"
+
+
+@dataclass
+class question_patcher:
+    responses: list[str]
+    next_response: int = 0
+
+    def __enter__(self):
+        global _question_asker
+        self._old_patcher = _question_asker
+        _question_asker = self.ask_question
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        global _question_patcher
+        _question_patcher = self._old_patcher
+
+    def ask_question(self, q: Question, response_type: type[T]) -> T:
+        q.application.output = DummyOutput()
+
+        def run(inp) -> T:
+            try:
+                input_response = self.responses[self.next_response]
+            except IndexError:
+                raise ValueError(
+                    f"Not enough responses provided. Expected {len(self.responses)}, got {self.next_response + 1} questions."
+                )
+            self.next_response += 1
+            inp.send_text(input_response + KeyInput.ENTER + "\r")
+            q.application.output = DummyOutput()
+            q.application.input = inp
+            return q.ask()
+
+        with create_pipe_input() as inp:
+            return run(inp)
+
+
+if __name__ == "__main__":
+    print(select_list("Select an option:", ["Option 1", "Option 2", "Option 3"]))  # noqa: T201
+    print(  # noqa: T201
+        select_dict(
+            "Select an option:",
+            {"Option 1": "1", "Option 2": "2", "Option 3": "3"},
+            default="Option 3",
+        )
+    )
+    print(confirm("Can you confirm?", default=True))  # noqa: T201
+    print(confirm("Can you confirm?", default=False))  # noqa: T201
+    print(  # noqa: T201
+        select_list_multiple("Select options:", ["Option 1", "Option 2", "Option 3"], ["Option 1"])
+    )
+    print(text("Enter your name:", default="John Doe"))  # noqa: T201