atlas-init 0.1.0__py3-none-any.whl → 0.1.4__py3-none-any.whl

atlas_init/__init__.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-VERSION = "0.1.0"
+VERSION = "0.1.4"
 
 
 def running_in_repo() -> bool:
@@ -8,5 +8,5 @@ def running_in_repo() -> bool:
     if py_directory.name != "py":
         return False
     git_directory = py_directory.parent / ".git"
-    fetch_head = git_directory / "FETCH_HEAD"
-    return git_directory.exists() and fetch_head.exists() and "atlas-init" in fetch_head.read_text()
+    git_config = git_directory / "config"
+    return git_directory.exists() and git_config.exists() and "atlas-init" in git_config.read_text()

atlas_init/atlas_init.yaml

@@ -1,69 +1,86 @@
 repo_aliases:
-    mongodb/terraform-provider-mongodbatlas: tf
-    mongodb/mongodbatlas-cloudformation-resources: cfn
+  mongodb/terraform-provider-mongodbatlas: tf
+  mongodb/mongodbatlas-cloudformation-resources: cfn
 test_suites:
 - name: cluster
   repo_go_packages:
     cfn:
-      - 'cfn-resources/cluster'
+    - cfn-resources/cluster
+- name: clusterm10
+  vars:
+    cluster_info_m10: true
 - name: federated
   repo_go_packages:
     tf:
-      - 'internal/service/federatedsettingsorgrolemapping'
+    - internal/service/federatedsettingsorgrolemapping
   vars:
     use_federated_vars: true
-- name: project
-  vars: 
-    use_project_extra: true
-  repo_go_packages:
-    cfn:
-    - 'cfn-resources/project'
-- name: network_peering
-  vars: 
-    use_aws_vpc: true
-    use_aws_vars: true
-    use_vpc_peering: true
+- name: network
   repo_go_packages:
     tf:
-    - 'internal/service/networkpeering'
-- name: network
-  vars: 
+    - internal/service/privatelinkendpointservicedatafederationonlinearchive
+  vars:
     use_private_link: true
+- name: network_peering
   repo_go_packages:
     tf:
-    - 'internal/service/privatelinkendpointservicedatafederationonlinearchive'
-- name: privatelink
+    - internal/service/networkpeering
   vars:
-    use_aws_vpc: true
+    use_vpc_peering: true
     use_aws_vars: true
+    use_aws_vpc: true
+- name: private_endpoint_regional_mode
   repo_go_packages:
     tf:
-    - 'internal/service/privatelinkendpointservice'
-- name: stream_connection
-  repo_go_packages: 
-    tf:
-    - 'internal/service/streamconnection'
+    - internal/service/privateendpointregionalmode
+  vars:
+    use_aws_vars: true
+- name: privatelink
+  repo_go_packages:
     cfn:
-    - 'cfn-resources/stream-connection'
+    - cfn-resources/cfn-private-endpoint-aws
+    tf:
+    - internal/service/privatelinkendpointservice
   vars:
-    cluster_info: true
-    stream_instance: true
-- name: search_index
+    use_aws_vars: true
+    use_aws_vpc: true
+- name: project
   repo_go_packages:
+    cfn:
+    - cfn-resources/project
     tf:
-    - 'internal/service/searchindex'
+    - internal/service/project
   vars:
-    cluster_info: true
+    use_project_extra: true
+- name: resource_policy
+  repo_go_packages:
+    cfn:
+    - cfn-resources/resource-policy
 - name: s3
   repo_go_packages:
     tf:
-    - 'internal/service/cloudbackupsnapshotexportbucket'
+    - internal/service/cloudbackupsnapshotexportbucket
   vars:
     use_aws_s3: true
 - name: s3_with_cluster
   repo_go_packages:
     tf:
-    - 'internal/service/cloudbackupsnapshotexportjob'
+    - internal/service/cloudbackupsnapshotexportjob
   vars:
+    cluster_info_m10: true
     use_aws_s3: true
+- name: search_index
+  repo_go_packages:
+    tf:
+    - internal/service/searchindex
+  vars:
+    cluster_info: true
+- name: stream_connection
+  repo_go_packages:
+    tf:
+    - internal/service/streamconnection
+    cfn:
+    - cfn-resources/stream-connection
+  vars:
     cluster_info_m10: true
+    stream_instance: true

atlas_init/cli.py

@@ -1,18 +1,19 @@
 import logging
-import sys
 from collections.abc import Callable
-from functools import partial
+from pathlib import Path
 from pydoc import locate
+from typing import Literal
 
 import typer
+from model_lib import dump, parse_payload
 from zero_3rdparty.file_utils import iter_paths
 
-from atlas_init import running_in_repo
-from atlas_init.cli_cfn.app import app as app_cfn
 from atlas_init.cli_helper import sdk_auto_changes
+from atlas_init.cli_helper.go import run_go_tests
 from atlas_init.cli_helper.run import (
     run_binary_command_is_ok,
     run_command_exit_on_failure,
+    run_command_receive_result,
 )
 from atlas_init.cli_helper.sdk import (
     SDK_VERSION_HELP,
@@ -31,7 +32,6 @@ from atlas_init.cli_helper.tf_runner import (
     get_tf_vars,
     run_terraform,
 )
-from atlas_init.cli_tf.app import app as app_tf
 from atlas_init.repos.go_sdk import go_sdk_breaking_changes
 from atlas_init.repos.path import (
     Repo,
@@ -43,11 +43,7 @@ from atlas_init.repos.path import (
 )
 from atlas_init.settings.config import RepoAliasNotFoundError
 from atlas_init.settings.env_vars import (
-    DEFAULT_PROFILE,
-    AtlasInitSettings,
     active_suites,
-    as_env_var_name,
-    env_var_names,
     init_settings,
 )
 from atlas_init.settings.path import (
@@ -55,57 +51,9 @@ from atlas_init.settings.path import (
     dump_vscode_dotenv,
     repo_path_rel_path,
 )
-from atlas_init.settings.rich_utils import configure_logging
+from atlas_init.typer_app import app, app_command, extra_root_commands
 
 logger = logging.getLogger(__name__)
-app = typer.Typer(name="atlas_init", invoke_without_command=True, no_args_is_help=True)
-app.add_typer(app_cfn, name="cfn")
-app.add_typer(app_tf, name="tf")
-
-app_command = partial(
-    app.command,
-    context_settings={"allow_extra_args": True, "ignore_unknown_options": True},
-)
-
-
-@app.callback(invoke_without_command=True)
-def main(
-    ctx: typer.Context,
-    log_level: str = typer.Option("INFO", help="use one of [INFO, WARNING, ERROR, CRITICAL]"),
-    profile: str = typer.Option(
-        DEFAULT_PROFILE,
-        "-p",
-        "--profile",
-        envvar=env_var_names("profile"),
-        help="used to load .env_manual, store terraform state and variables, and dump .env files.",
-    ),
-    project_name: str = typer.Option(
-        "",
-        "--project",
-        envvar=env_var_names("project_name"),
-        help="atlas project name to create",
-    ),
-):
-    explicit_env_vars: dict[str, str] = {}
-    if project_name != "":
-        explicit_env_vars[as_env_var_name("project_name")] = project_name
-    configure_logging(log_level)
-    logger.info(f"running in repo: {running_in_repo()} python location:{sys.executable}")
-    missing_env_vars, ambiguous_env_vars = AtlasInitSettings.check_env_vars(
-        profile,
-        required_extra_fields=["project_name"],
-        explicit_env_vars=explicit_env_vars,
-    )
-    if missing_env_vars:
-        typer.echo(f"missing env_vars: {missing_env_vars}")
-    if ambiguous_env_vars:
-        typer.echo(
-            f"amiguous env_vars: {missing_env_vars} (specified both in cli & in .env-manual file with different values)"
-        )
-    if missing_env_vars or ambiguous_env_vars:
-        raise typer.Exit(1)
-    command = ctx.invoked_subcommand
-    logger.info(f"in the app callback, log-level: {log_level}, command: {command}")
 
 
 @app_command()
@@ -116,12 +64,19 @@ def init(context: typer.Context):
     run_terraform(settings, "init", extra_args)
 
 
+@app_command()
+def plan(context: typer.Context, *, skip_outputs: bool = False):
+    _plan_or_apply(context.args, "plan", skip_outputs=skip_outputs)
+
+
 @app_command()
 def apply(context: typer.Context, *, skip_outputs: bool = False):
+    _plan_or_apply(context.args, "apply", skip_outputs=skip_outputs)
+
+
+def _plan_or_apply(extra_args: list[str], command: Literal["plan", "apply"], *, skip_outputs: bool):
     settings = init_settings()
-    extra_args = context.args
-    logger.info(f"apply extra args: {extra_args}")
-    logger.info("in the apply command")
+    logger.info(f"using the '{command}' command, extra args: {extra_args}")
     try:
         suites = active_suites(settings)
     except (CwdIsNoRepoPathError, RepoAliasNotFoundError) as e:
@@ -132,10 +87,10 @@ def apply(context: typer.Context, *, skip_outputs: bool = False):
     dump_tf_vars(settings, tf_vars)
 
     try:
-        run_terraform(settings, "apply", extra_args)
+        run_terraform(settings, command, extra_args)
     except TerraformRunError as e:
         logger.error(repr(e))  # noqa: TRY400
-        return
+        raise typer.Exit(1) from e
 
     if not skip_outputs:
         export_outputs(settings)
@@ -149,6 +104,9 @@ def apply(context: typer.Context, *, skip_outputs: bool = False):
 def destroy(context: typer.Context):
     extra_args = context.args
     settings = init_settings()
+    if not settings.tf_state_path.exists():
+        logger.warning(f"no terraform state found  {settings.tf_state_path}, exiting")
+        return
     tf_vars = get_tf_vars(settings, [])
     dump_tf_vars(settings, tf_vars)
     try:
@@ -164,9 +122,15 @@ def test_go():
     suites = active_suites(settings)
     sorted_suites = sorted(suite.name for suite in suites)
     logger.info(f"running go tests for {len(suites)} test-suites: {sorted_suites}")
-    raise NotImplementedError("fix me later!")  # noqa
-    # package_prefix = settings.config.go_package_prefix(repo_alias)
-    # run_go_tests(repo_path, repo_alias, package_prefix, settings, active_suites)
+    match repo_alias := current_repo():
+        case Repo.CFN:
+            raise NotImplementedError
+        case Repo.TF:
+            repo_path = current_repo_path()
+            package_prefix = settings.config.go_package_prefix(repo_alias)
+            run_go_tests(repo_path, repo_alias, package_prefix, settings, suites)
+        case _:
+            raise NotImplementedError
 
 
 @app_command()
@@ -186,8 +150,10 @@ def sdk_upgrade(
 
     sdk_breaking_changes_path = go_sdk_breaking_changes(repo_path)
     all_breaking_changes = parse_breaking_changes(sdk_breaking_changes_path, old, new)
-    replace_in = f"go.mongodb.org/atlas-sdk/{old}/admin"
-    replace_out = f"go.mongodb.org/atlas-sdk/{new}/admin"
+    replacements = {
+        f"go.mongodb.org/atlas-sdk/{old}/mockadmin": f"go.mongodb.org/atlas-sdk/{new}/mockadmin",
+        f"go.mongodb.org/atlas-sdk/{old}/admin": f"go.mongodb.org/atlas-sdk/{new}/admin",
+    }
     auto_modifier: Callable[[str, str], str] | None = None
     if auto_change_name:
         func_path = f"{sdk_auto_changes.__name__}.{auto_change_name}"
@@ -198,7 +164,7 @@ def sdk_upgrade(
     resources_breaking_changes: set[str] = set()
     for path in iter_paths(repo_path, "*.go", ".mockery.yaml"):
         text_old = path.read_text()
-        if replace_in not in text_old:
+        if all(replace_in not in text_old for replace_in in replacements):
             continue
         r_name = resource_name(repo_path, path)
         if resource and resource != r_name:
@@ -210,7 +176,9 @@ def sdk_upgrade(
             logger.warning(f"found breaking changes: {changes_formatted}")
             if is_removed(breaking_changes):
                 resources_breaking_changes.add(r_name)
-        text_new = text_old.replace(replace_in, replace_out)
+        text_new = text_old
+        for replace_in, replace_out in replacements.items():
+            text_new = text_new.replace(replace_in, replace_out)
         if not dry_run:
             if auto_modifier:
                 text_new = auto_modifier(text_new, old)
@@ -237,18 +205,22 @@ def sdk_upgrade(
 def pre_commit(
     skip_build: bool = typer.Option(default=False),
     skip_lint: bool = typer.Option(default=False),
+    max_issues: int = typer.Option(1000, "-m", "--max"),
 ):
+    golang_ci_lint_args = f"--max-same-issues {max_issues} --max-issues-per-linter {max_issues}"
     match current_repo():
         case Repo.CFN:
             repo_path, resource_path, r_name = find_paths()
             build_cmd = f"cd {resource_path} && make build"
             # TODO: understand why piping to grep doesn't work
             # f"golangci-lint run --path-prefix=./cfn-resources | grep {r_name}"
-            format_cmd_str = "cd cfn-resources && golangci-lint run --path-prefix=./cfn-resources"
+            format_cmd_str = (
+                f"cd cfn-resources && golangci-lint run --path-prefix=./cfn-resources {golang_ci_lint_args}"
+            )
         case Repo.TF:
             repo_path = current_repo_path()
             build_cmd = "make build"
-            format_cmd_str = "golangci-lint run"
+            format_cmd_str = f"golangci-lint run {golang_ci_lint_args}"
         case _:
             raise NotImplementedError
     if skip_build:
@@ -261,7 +233,39 @@ def pre_commit(
         run_command_exit_on_failure(format_cmd_str, cwd=repo_path, logger=logger)
 
 
+@app_command()
+def repo_dump():
+    code_root = Path.home() / "code"
+    path_urls = {}
+    for repo_git_path in iter_paths(code_root, ".git", exclude_folder_names=[".terraform"]):
+        repo_path = repo_git_path.parent
+        logger.info(f"repo: {repo_path}")
+        url = run_command_receive_result("git remote get-url origin", cwd=repo_path, logger=logger, can_fail=True)
+        if url.startswith("FAIL:"):
+            continue
+        path_urls[str(repo_path)] = url
+    out_path = code_root / "repos.json"
+    repos_json = dump(path_urls, "pretty_json")
+    out_path.write_text(repos_json)
+
+
+@app_command()
+def repo_clone():
+    repos_file = Path.home() / "code" / "repos.json"
+    repo_path_json: dict[str, str] = parse_payload(repos_file)  # type: ignore
+    for repo_path_str, url in repo_path_json.items():
+        logger.info(f"cloning {url} @ {repo_path_str}")
+        repo_path = Path(repo_path_str)
+        parent_dir = repo_path.parent
+        parent_dir.mkdir(parents=True, exist_ok=True)
+        if repo_path.exists():
+            logger.warning(f"skipping {repo_path}, already exists")
+            continue
+        run_command_exit_on_failure(f"git clone {url} {repo_path.name}", cwd=parent_dir, logger=logger)
+
+
 def typer_main():
+    extra_root_commands()
     app()
 
 

atlas_init/cli_cfn/app.py

@@ -1,42 +1,35 @@
 import logging
 import os
-from pathlib import Path
 
 import typer
-from model_lib import dump, parse_payload
-from rich import prompt
+from model_lib import parse_payload
 from zero_3rdparty.file_utils import clean_dir
 
-from atlas_init.cli_args import parse_key_values, parse_key_values_any
-from atlas_init.cli_cfn.cfn import (
+from atlas_init.cli_cfn.aws import (
     activate_resource_type,
-    create_stack,
     deactivate_third_party_type,
-    delete_stack,
     deregister_cfn_resource_type,
     get_last_cfn_type,
-    update_stack,
+    wait_on_stack_ok,
+)
+from atlas_init.cli_cfn.aws import (
+    delete_stack as delete_stack_aws,
 )
 from atlas_init.cli_cfn.cfn_parameter_finder import (
-    check_execution_role,
-    decode_parameters,
-    infer_template_path,
     read_execution_role,
 )
+from atlas_init.cli_cfn.example import example_cmd
+from atlas_init.cli_cfn.files import create_sample_file, has_md_link, iterate_schemas
 from atlas_init.cli_helper.run import run_command_is_ok
 from atlas_init.cloud.aws import run_in_regions
 from atlas_init.repos.cfn import (
-    CfnOperation,
-    CfnType,
-    Operation,
-    infer_cfn_type_name,
     validate_type_name_regions,
 )
-from atlas_init.repos.path import Repo, current_dir, find_paths
+from atlas_init.repos.path import Repo, current_dir, find_paths, resource_root
 from atlas_init.settings.env_vars import active_suites, init_settings
-from atlas_init.settings.interactive import confirm
 
 app = typer.Typer(no_args_is_help=True)
+app.command(name="example")(example_cmd)
 logger = logging.getLogger(__name__)
 
 
@@ -92,104 +85,6 @@ def dereg(
         run_in_regions(deactivate, regions)
 
 
-@app.command()
-def example(
-    type_name: str = typer.Argument(default_factory=infer_cfn_type_name),
-    region: str = typer.Argument(...),
-    stack_name: str = typer.Argument(...),
-    operation: str = typer.Argument(...),
-    params: list[str] = typer.Option(..., "-p", default_factory=list),
-    resource_params: list[str] = typer.Option(..., "-r", default_factory=list),
-    stack_timeout_s: int = typer.Option(300, "-t", "--stack-timeout-s"),
-):
-    params_parsed: dict[str, str] = {}
-    if params:
-        params_parsed = parse_key_values(params)
-    resource_params_parsed = {}
-    if resource_params:
-        resource_params_parsed = parse_key_values_any(resource_params)
-        if resource_params_parsed:
-            logger.info(f"using resource params: {resource_params_parsed}")
-    logger.info(f"about to update stack {stack_name} for {type_name} in {region} with {operation}, params: {params}")
-    settings = init_settings()
-    type_name, region = CfnType.validate_type_region(type_name, region)  # type: ignore
-    CfnOperation(operaton=operation)  # type: ignore
-    repo_path, resource_path, _ = find_paths(Repo.CFN)
-    env_vars_generated = settings.load_env_vars_generated()
-    cfn_execution_role = check_execution_role(repo_path, env_vars_generated)
-
-    cfn_type_details = get_last_cfn_type(type_name, region, is_third_party=False)
-    logger.info(f"found cfn_type_details {cfn_type_details} for {type_name}")
-    submit_cmd = f"cfn submit --verbose --set-default --region {region} --role-arn {cfn_execution_role}"
-    if cfn_type_details is None and confirm(
-        f"No existing {type_name} found, ok to run:\n{submit_cmd}\nsubmit?",
-        is_interactive=settings.is_interactive,
-        default=True,
-    ):
-        assert run_command_is_ok(cmd=submit_cmd.split(), env=None, cwd=resource_path, logger=logger)
-        cfn_type_details = get_last_cfn_type(type_name, region, is_third_party=False)
-    assert cfn_type_details, f"no cfn_type_details found for {type_name}"
-
-    if operation == Operation.DELETE:
-        delete_stack(region, stack_name)
-        return
-    template_path = infer_template_path(repo_path, type_name, stack_name)
-    template_path, parameters, not_found = decode_parameters(
-        exported_env_vars=env_vars_generated,
-        template_path=template_path,
-        stack_name=stack_name,
-        force_params=params_parsed,
-        resource_params=resource_params_parsed,
-        type_name=type_name,
-    )
-    logger.info(f"parameters: {parameters}")
-    if not_found:
-        # TODO: support specifying these extra
-        logger.critical(f"need to fill out parameters manually: {not_found} for {type_name}")
-        raise typer.Exit(1)
-    if not prompt.Confirm("parameters 👆looks good?")():
-        raise typer.Exit(1)
-    if operation == Operation.CREATE:
-        create_stack(
-            stack_name,
-            template_str=template_path.read_text(),
-            region_name=region,
-            role_arn=cfn_execution_role,
-            parameters=parameters,
-            timeout_seconds=stack_timeout_s,
-        )
-    elif operation == Operation.UPDATE:
-        update_stack(
-            stack_name,
-            template_str=template_path.read_text(),
-            region_name=region,
-            parameters=parameters,
-            role_arn=cfn_execution_role,
-            timeout_seconds=stack_timeout_s,
-        )
-    else:
-        raise NotImplementedError
-
-
-def _create_sample_file(
-    samples_file: Path,
-    log_group_name: str,
-    resource_state: dict,
-    prev_resource_state: dict | None = None,
-):
-    logger.info(f"adding sample @ {samples_file}")
-    assert isinstance(resource_state, dict)
-    new_json = dump(
-        {
-            "providerLogGroupName": log_group_name,
-            "previousResourceState": prev_resource_state or {},
-            "desiredResourceState": resource_state,
-        },
-        "json",
-    )
-    samples_file.write_text(new_json)
-
-
 @app.command()
 def inputs(
     context: typer.Context,
@@ -240,11 +135,11 @@ def inputs(
         assert isinstance(resource_state, dict), f"input file with not a dict {resource_state}"
         samples_file = samples_dir / file.name
         if file.name.endswith("_create.json"):
-            _create_sample_file(samples_file, log_group_name, resource_state)
+            create_sample_file(samples_file, log_group_name, resource_state)
         if file.name.endswith("_update.json"):
             prev_state_path = file.parent / file.name.replace("_update.json", "_create.json")
             prev_state: dict = parse_payload(prev_state_path)  # type: ignore
-            _create_sample_file(
+            create_sample_file(
                 samples_file,
                 log_group_name,
                 resource_state,
@@ -256,3 +151,31 @@ def inputs(
             new_filename = inputs_dir / new_name
             file.rename(new_filename)
             logger.info(f"renamed from {file} -> {new_filename}")
+
+
+@app.command()
+def gen_docs():
+    repo_path, *_ = find_paths(Repo.CFN)
+    root = resource_root(repo_path)
+    for path, schema in iterate_schemas(root):
+        if has_md_link(schema.description):
+            logger.warning(f"found md link in {schema.type_name} in {path}")
+
+
+@app.command()
+def wait_on_stack(
+    stack_name: str = typer.Argument(...),
+    region: str = typer.Argument(...),
+    timeout_s: int = typer.Option(300, "-t", "--timeout-seconds"),
+):
+    wait_on_stack_ok(stack_name, region, timeout_seconds=timeout_s)
+    logger.info(f"stack {stack_name} in {region} is ready ✅")
+
+
+@app.command()
+def delete_stack(
+    stack_name: str = typer.Argument(...),
+    region: str = typer.Argument(...),
+):
+    delete_stack_aws(region, stack_name)
+    logger.info(f"stack {stack_name} in {region} is deleted ✅")