atlas-init 0.4.5__py3-none-any.whl → 0.6.0__py3-none-any.whl

atlas_init/__init__.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-VERSION = "0.4.5"
+VERSION = "0.6.0"
 
 
 def running_in_repo() -> bool:

atlas_init/cli.py

@@ -5,6 +5,7 @@ from pydoc import locate
 from typing import Literal
 
 import typer
+from ask_shell.typer_command import configure_logging
 from model_lib import dump, parse_payload
 from zero_3rdparty.file_utils import iter_paths
 
@@ -255,6 +256,7 @@ def repo_clone():
 
 def typer_main():
     extra_root_commands()
+    configure_logging(app)
     app()
 
 

atlas_init/cli_cfn/cfn_parameter_finder.py

@@ -8,12 +8,14 @@ from mypy_boto3_cloudformation.type_defs import ParameterTypeDef
 from pydantic import ConfigDict, Field
 from rich import prompt
 from zero_3rdparty.file_utils import clean_dir
+from zero_3rdparty.dict_nested import iter_nested_key_values, update
 
 from atlas_init.cli_cfn.files import create_sample_file, default_log_group_name
 from atlas_init.cloud.aws import PascalAlias
 from atlas_init.repos.cfn import CfnType, cfn_examples_dir, cfn_type_normalized
 
 logger = logging.getLogger(__name__)
+UNKNOWN_PLACEHOLDER = "UNKNOWN"
 
 
 class TemplatePathNotFoundError(Exception):
@@ -35,21 +37,25 @@ def infer_template_path(repo_path: Path, type_name: str, stack_name: str, exampl
     if not template_paths:
         raise TemplatePathNotFoundError(type_name, examples_dir)
     if len(template_paths) > 1:
-        expected_folder = cfn_type_normalized(type_name)
-        if (expected_folders := [p for p in template_paths if p.parent.name == expected_folder]) and len(
-            expected_folders
-        ) == 1:
-            logger.info(f"using template: {expected_folders[0]}")
-            return expected_folders[0]
-        choices = {p.stem: p for p in template_paths}
-        if stack_path := choices.get(stack_name):
-            logger.info(f"using template @ {stack_path} based on stack name: {stack_name}")
-            return stack_path
-        selected_path = prompt.Prompt("Choose example template: ", choices=list(choices))()
-        return choices[selected_path]
+        return choose_template_path(type_name, template_paths, stack_name)
     return template_paths[0]
 
 
+def choose_template_path(type_name: str, template_paths: list[Path], stack_name: str) -> Path:
+    expected_folder = cfn_type_normalized(type_name)
+    if (expected_folders := [p for p in template_paths if p.parent.name == expected_folder]) and len(
+        expected_folders
+    ) == 1:
+        logger.info(f"using template: {expected_folders[0]}")
+        return expected_folders[0]
+    choices = {p.stem: p for p in template_paths}
+    if stack_path := choices.get(stack_name):
+        logger.info(f"using template @ {stack_path} based on stack name: {stack_name}")
+        return stack_path
+    selected_path = prompt.Prompt("Choose example template: ", choices=list(choices))()
+    return choices[selected_path]
+
+
 parameters_exported_env_vars = {
     "OrgId": "MONGODB_ATLAS_ORG_ID",
     "Profile": "ATLAS_INIT_CFN_PROFILE",
@@ -122,10 +128,6 @@ class CfnTemplate(Entity):
         assert self.find_resource(type_name)
         return cfn_type_normalized(type_name)
 
-    def add_resource_params(self, type_name: str, resources: dict[str, Any]):
-        resource = self.find_resource(type_name)
-        resource.properties.update(resources)
-
     def get_resource_properties(self, type_name: str, parameters: list[ParameterTypeDef]) -> dict:
         resource = self.find_resource(type_name)
         properties = resource.properties
@@ -152,40 +154,30 @@ class CfnTemplate(Entity):
         return properties
 
 
-def updated_template_path(path: Path) -> Path:
-    old_stem = path.stem
-    new_name = path.name.replace(old_stem, f"{old_stem}-updated")
-    return path.with_name(new_name)
+class CfnTemplateUnknownParametersError(Exception):
+    def __init__(self, unknown_params: list[str]) -> None:
+        self.unknown_params = unknown_params
 
 
-def decode_parameters(
-    template_path: Path,
-    type_name: str,
-    stack_name: str,
-    force_params: dict[str, Any] | None = None,
-    resource_params: dict[str, Any] | None = None,
-) -> tuple[Path, list[ParameterTypeDef], set[str]]:
-    cfn_template = parse_model(template_path, t=CfnTemplate)
-    if resource_params:
-        cfn_template.add_resource_params(type_name, resource_params)
-        template_path = updated_template_path(template_path)
-        logger.info(f"updating template {template_path} with {resource_params}")
-        raw_dict = cfn_template.model_dump(by_alias=True, exclude_unset=True)
-        file_extension = template_path.suffix.lstrip(".")
-        dump_format = "pretty_json" if file_extension == "json" else file_extension
-        template_str = dump(raw_dict, format=dump_format)
-        template_path.write_text(template_str)
-    parameters_dict: dict[str, Any] = {}
+def infer_template_parameters(
+    path: Path, type_name: str, stack_name: str, explicit_params: dict[str, Any]
+) -> list[ParameterTypeDef]:
+    cfn_template = parse_model(path, t=CfnTemplate)
+    parameters_dict: dict[str, Any] = {key: UNKNOWN_PLACEHOLDER for key in cfn_template.parameters.keys()}
     type_defaults = type_names_defaults.get(cfn_template.normalized_type_name(type_name), {})
     if stack_name_param := type_defaults.pop(STACK_NAME_PARAM, None):
         type_defaults[stack_name_param] = stack_name
-
     for param_name, param in cfn_template.parameters.items():
+        explicit_value = explicit_params.get(param_name)
+        if explicit_value is not None:
+            logger.info(f"using explicit value for {param_name}={explicit_value}")
+            parameters_dict[param_name] = explicit_value
+            continue
         if type_default := type_defaults.get(param_name):
             logger.info(f"using type default for {param_name}={type_default}")
             parameters_dict[param_name] = type_default
             continue
-        if env_key := parameters_exported_env_vars.get(param_name):  # noqa: SIM102
+        if env_key := parameters_exported_env_vars.get(param_name):
             if env_value := os.environ.get(env_key):
                 logger.info(f"using {env_key} to fill parameter: {param_name}")
                 parameters_dict[param_name] = env_value
@@ -195,19 +187,12 @@ def decode_parameters(
             parameters_dict[param_name] = "false"
             continue
         if default := param.default:
+            logger.info(f"using default for {param_name}={default}")
             parameters_dict[param_name] = default
             continue
-        logger.warning(f"unable to auto-filll param: {param_name}")
-        parameters_dict[param_name] = "UNKNOWN"
-
-    if force_params:
-        logger.warning(f"overiding params: {force_params} for {stack_name}")
-        parameters_dict |= force_params
-    unknown_params = {key for key, value in parameters_dict.items() if value == "UNKNOWN"}
-    parameters: list[ParameterTypeDef] = [
-        {"ParameterKey": key, "ParameterValue": value} for key, value in parameters_dict.items()
-    ]
-    return template_path, parameters, unknown_params
+    if unknown_params := {key for key, value in parameters_dict.items() if value == UNKNOWN_PLACEHOLDER}:
+        raise CfnTemplateUnknownParametersError(sorted(unknown_params))
+    return [{"ParameterKey": key, "ParameterValue": value} for key, value in parameters_dict.items()]
 
 
 def dump_resource_to_file(
@@ -240,3 +225,26 @@ def dump_sample_file(
         prev_resource_state={},
     )
     return samples_path
+
+
+def modify_resource_with_params(resource: CfnResource, resource_params: dict[str, Any]) -> None:
+    updates: dict[str, tuple[str, Any]] = {}
+    resource_properties = resource.properties
+    for path, value in iter_nested_key_values(resource_properties, include_list_indexes=True):
+        if not isinstance(value, dict):
+            continue
+        if "Ref" not in value:
+            continue
+        param_name = value["Ref"]
+        assert isinstance(param_name, str), f"Ref must be a string, {path}, got={param_name!r}"
+        if param_value := resource_params.get(param_name):
+            updates[param_name] = (path, param_value)
+        else:
+            logger.warning(f"unable to find parameter {param_name} in resource params, path={path}")
+
+    for param_name, param_value in resource_params.items():
+        if update_path_value := updates.get(param_name):
+            update(resource_properties, *update_path_value)
+        else:
+            logger.warning(f"No ref found for {param_name} assumming top level on resource")
+            resource_properties[param_name] = param_value

atlas_init/cli_cfn/example.py

@@ -15,9 +15,9 @@ from atlas_init.cli_cfn.aws import (
 from atlas_init.cli_cfn.aws import delete_stack as delete_stack_aws
 from atlas_init.cli_cfn.cfn_parameter_finder import (
     CfnTemplate,
-    decode_parameters,
     dump_resource_to_file,
     dump_sample_file,
+    infer_template_parameters,
     infer_template_path,
 )
 from atlas_init.repos.cfn import CfnType, Operation, infer_cfn_type_name
@@ -142,6 +142,13 @@ def example_handler(
     delete_first = inputs.delete_stack_first
     force_deregister = inputs.force_deregister
     execution_role = inputs.execution_role
+
+    template_path = infer_template_path(repo_path, type_name, stack_name, inputs.example_name)
+    parameters = infer_template_parameters(template_path, type_name, stack_name, inputs.resource_params or {})
+    logger.info(f"parameters: {parameters}")
+    if not prompt.Confirm("parameters 👆looks good?")():
+        raise typer.Abort
+
     logger.info(f"using execution role: {execution_role}")
     if not inputs.is_export and not inputs.force_keep:
         ensure_resource_type_activated(
@@ -157,14 +164,6 @@ def example_handler(
         delete_stack_aws(region, stack_name, execution_role)
         if not delete_first:
             return
-    template_path = infer_template_path(repo_path, type_name, stack_name, inputs.example_name)
-    template_path, parameters, not_found = decode_parameters(
-        template_path=template_path,
-        stack_name=stack_name,
-        force_params=inputs.resource_params,
-        resource_params=inputs.resource_params,
-        type_name=type_name,
-    )
     if inputs.register_all_types_in_example:
         extra_example_types = [t for t in CfnTemplate.read_template_types(template_path) if t != type_name]
         for extra_type in extra_example_types:
@@ -177,13 +176,6 @@ def example_handler(
                 resource_path,
                 execution_role,
             )
-    logger.info(f"parameters: {parameters}")
-    if not_found:
-        # TODO: support specifying these extra
-        logger.critical(f"need to fill out parameters manually: {not_found} for {type_name}")
-        raise typer.Exit(1)
-    if not prompt.Confirm("parameters 👆looks good?")():
-        raise typer.Abort
     if inputs.export_example_to_inputs:
         out_inputs = dump_resource_to_file(resource_path / "inputs", template_path, type_name, parameters)
         logger.info(f"dumped to {out_inputs} ✅")

atlas_init/cli_helper/go.py

@@ -9,10 +9,8 @@ from pydantic import Field
 
 from atlas_init.cli_helper.run import run_command_is_ok_output
 from atlas_init.cli_tf.go_test_run import (
-    GoTestContext,
-    GoTestContextStep,
     GoTestRun,
-    parse,
+    parse_tests,
 )
 from atlas_init.settings.config import TestSuite
 from atlas_init.settings.env_vars import AtlasInitSettings
@@ -179,7 +177,7 @@ def resolve_env_vars(
     test_env_vars |= {
         "TF_ACC": "1",
         "TF_LOG": "DEBUG",
-        "MONGODB_ATLAS_PREVIEW_PROVIDER_V2_ADVANCED_CLUSTER": "false" if use_old_schema else "true",
+        "MONGODB_ATLAS_PREVIEW_PROVIDER_V2_ADVANCED_CLUSTER": ("false" if use_old_schema else "true"),
     }
     test_env_vars |= env_vars_for_capture(capture_mode)
     logger.info(f"go test env-vars-extra: {sorted(test_env_vars)}")
@@ -231,17 +229,15 @@ def _run_tests(
             logger.exception(f"failed to run command for {name}")
             results.failure_names.add(name)
             continue
-        context = GoTestContext(
-            name=name,
-            html_url=f"file://{_log_path(logs_dir, name)}",
-            steps=[GoTestContextStep(name="local-run")],
-        )
         try:
-            parsed_tests = list(parse(command_out.splitlines(), context, test_step_nr=0))
+            parsed_tests = parse_tests(command_out.splitlines())
         except Exception:
             logger.exception(f"failed to parse tests for {name}")
             results.failure_names.add(name)
             continue
+        for test in parsed_tests:
+            test.log_path = _log_path(logs_dir, name)
+            # todo: possible add other fields
         if not parsed_tests and not ok:
             results.failure_names.add(name)
             logger.error(f"failed to run tests for {name}: {command_out}")

atlas_init/cli_root/mms_released.py

@@ -0,0 +1,46 @@
+import logging
+from pathlib import Path
+
+import requests
+import typer
+from atlas_init.typer_app import app_command
+from git import Repo
+
+logger = logging.getLogger(__name__)
+
+
+@app_command()
+def mms_released(
+    mms_repo: Path = typer.Option(..., "-r", "--mms-repo", help="the path to the mms repo"),
+    commit_shas: list[str] = typer.Option(
+        ..., "-c", "--commit-shas", help="the commit shas to check for release, can be set multiple times"
+    ),
+    sha_url: str = typer.Option(
+        "https://cloud.mongodb.com/api/private/unauth/version",
+        "-u",
+        "--url",
+        help="the url to get the current sha from",
+    ),
+):
+    assert mms_repo.exists(), f"mms repo not found @ {mms_repo}"
+    git_repo = Repo(mms_repo)
+    git_repo.git.fetch("origin")
+    for sha in commit_shas:
+        assert git_repo.commit(sha), f"commit {sha} not found in {mms_repo}"
+    current_sha_response = requests.get(sha_url, timeout=10)
+    current_sha_response.raise_for_status()
+    current_sha = current_sha_response.text.strip()
+    assert current_sha, f"unable to get current sha from {current_sha_response.url}"
+    logger.info(f"current sha of prod: {current_sha}")
+    assert git_repo.commit(current_sha)
+    remaining_shas = set(commit_shas)
+    for commit in git_repo.iter_commits(rev=current_sha):
+        commit_sha = commit.hexsha
+        if commit_sha in commit_shas:
+            commit_message = commit.message.rstrip("\n")  # type: ignore
+            logger.info(f"found commit {commit_sha} with message {commit_message}")
+            remaining_shas.remove(commit_sha)
+        if not remaining_shas:
+            logger.info("all commits found ✅")
+            return
+    logger.info(f"remaining shas: {','.join(sorted(remaining_shas))} ❌")

atlas_init/cli_tf/app.py

@@ -1,34 +1,18 @@
 import logging
-import os
 import sys
-from collections import defaultdict
-from datetime import timedelta
 from pathlib import Path
 
 import typer
-from zero_3rdparty.datetime_utils import utc_now
 from zero_3rdparty.file_utils import clean_dir
 
 from atlas_init.cli_args import option_sdk_repo_path
 from atlas_init.cli_helper.run import (
-    add_to_clipboard,
     run_binary_command_is_ok,
     run_command_exit_on_failure,
-    run_command_receive_result,
 )
+from atlas_init.cli_tf.ci_tests import ci_tests
 from atlas_init.cli_tf.changelog import convert_to_changelog
 from atlas_init.cli_tf.example_update import update_example_cmd
-from atlas_init.cli_tf.github_logs import (
-    GH_TOKEN_ENV_NAME,
-    find_test_runs,
-    include_filestems,
-    include_test_jobs,
-)
-from atlas_init.cli_tf.go_test_run import GoTestRun
-from atlas_init.cli_tf.go_test_summary import (
-    create_detailed_summary,
-    create_short_summary,
-)
 from atlas_init.cli_tf.log_clean import log_clean
 from atlas_init.cli_tf.mock_tf_log import mock_tf_log_cmd
 from atlas_init.cli_tf.schema import (
@@ -41,7 +25,7 @@ from atlas_init.cli_tf.schema_v2 import (
     generate_resource_go_resource_schema,
     parse_schema,
 )
-from atlas_init.cli_tf.schema_v2_api_parsing import add_api_spec_info
+from atlas_init.cli_tf.openapi import add_api_spec_info
 from atlas_init.cli_tf.schema_v2_sdk import generate_model_go, parse_sdk_model
 from atlas_init.repos.go_sdk import download_admin_api
 from atlas_init.repos.path import Repo, current_repo_path
@@ -52,6 +36,7 @@ app = typer.Typer(no_args_is_help=True)
 app.command(name="mock-tf-log")(mock_tf_log_cmd)
 app.command(name="example-update")(update_example_cmd)
 app.command(name="log-clean")(log_clean)
+app.command(name="ci-tests")(ci_tests)
 logger = logging.getLogger(__name__)
 
 
@@ -153,72 +138,6 @@ def example_gen(
         file_utils.copy(path, dest_path, clean_dest=False)
 
 
-@app.command()
-def ci_tests(
-    test_group_name: str = typer.Option("", "-g"),
-    max_days_ago: int = typer.Option(1, "-d", "--days"),
-    branch: str = typer.Option("master", "-b", "--branch"),
-    workflow_file_stems: str = typer.Option("test-suite,terraform-compatibility-matrix", "-w", "--workflow"),
-    only_last_workflow: bool = typer.Option(False, "-l", "--last"),
-    names: str = typer.Option(
-        "",
-        "-n",
-        "--test-names",
-        help="comma separated list of test names to filter, e.g., TestAccCloudProviderAccessAuthorizationAzure_basic,TestAccBackupSnapshotExportBucket_basicAzure",
-    ),
-    summary_name: str = typer.Option(
-        "",
-        "-s",
-        "--summary",
-        help="the name of the summary directory to store detailed test results",
-    ),
-):  # sourcery skip: use-named-expression
-    names_set: set[str] = set()
-    if names:
-        names_set.update(names.split(","))
-        logger.info(f"filtering tests by names: {names_set}")
-    repo_path = current_repo_path(Repo.TF)
-    token = run_command_receive_result("gh auth token", cwd=repo_path, logger=logger)
-    os.environ[GH_TOKEN_ENV_NAME] = token
-    end_test_date = utc_now()
-    start_test_date = end_test_date - timedelta(days=max_days_ago)
-    job_runs = find_test_runs(
-        start_test_date,
-        include_job=include_test_jobs(test_group_name),
-        branch=branch,
-        include_workflow=include_filestems(set(workflow_file_stems.split(","))),
-    )
-    test_results: dict[str, list[GoTestRun]] = defaultdict(list)
-    workflow_ids = set()
-    for key in sorted(job_runs.keys(), reverse=True):
-        workflow_id, job_id = key
-        workflow_ids.add(workflow_id)
-        if only_last_workflow and len(workflow_ids) > 1:
-            logger.info("only showing last workflow")
-            break
-        runs = job_runs[key]
-        if not runs:
-            logger.warning(f"no go tests for job_id={job_id}")
-            continue
-        for run in runs:
-            test_name = run.name
-            if names_set and test_name not in names_set:
-                continue
-            test_results[test_name].append(run)
-
-    if summary_name:
-        summary = create_detailed_summary(summary_name, end_test_date, start_test_date, test_results, names_set)
-    else:
-        failing_names = [name for name, name_runs in test_results.items() if all(run.is_failure for run in name_runs)]
-        if not failing_names:
-            logger.info("ALL TESTS PASSED! ✅")
-            return
-        summary = create_short_summary(test_results, failing_names)
-    summary_str = "\n".join(summary)
-    add_to_clipboard(summary_str, logger)
-    logger.info(summary_str)
-
-
 @app.command()
 def schema2(
     resource: str = typer.Argument(