atlas-init 0.3.1__py3-none-any.whl → 0.3.3__py3-none-any.whl

atlas_init/__init__.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-VERSION = "0.3.1"
+VERSION = "0.3.3"
 
 
 def running_in_repo() -> bool:

atlas_init/atlas_init.yaml

@@ -6,6 +6,10 @@ test_suites:
   repo_go_packages:
     cfn:
     - cfn-resources/cluster
+- name: cluster_sdk
+  repo_go_packages:
+    tf:
+    - internal/service/advancedcluster
 - name: clusterm10
   vars:
     cluster_info_m10: true
@@ -56,6 +60,8 @@ test_suites:
   repo_go_packages:
     cfn:
     - cfn-resources/resource-policy
+    tf:
+    - internal/service/resourcepolicy
 - name: s3
   repo_go_packages:
     tf:

atlas_init/cli.py

@@ -9,7 +9,6 @@ from model_lib import dump, parse_payload
 from zero_3rdparty.file_utils import iter_paths
 
 from atlas_init.cli_helper import sdk_auto_changes
-from atlas_init.cli_helper.go import run_go_tests
 from atlas_init.cli_helper.run import (
     run_binary_command_is_ok,
     run_command_exit_on_failure,
@@ -116,23 +115,6 @@ def destroy(context: typer.Context):
         return
 
 
-@app_command()
-def test_go():
-    settings = init_settings()
-    suites = active_suites(settings)
-    sorted_suites = sorted(suite.name for suite in suites)
-    logger.info(f"running go tests for {len(suites)} test-suites: {sorted_suites}")
-    match repo_alias := current_repo():
-        case Repo.CFN:
-            raise NotImplementedError
-        case Repo.TF:
-            repo_path = current_repo_path()
-            package_prefix = settings.config.go_package_prefix(repo_alias)
-            run_go_tests(repo_path, repo_alias, package_prefix, settings, suites)
-        case _:
-            raise NotImplementedError
-
-
 @app_command()
 def sdk_upgrade(
     old: SdkVersion = typer.Argument(help=SDK_VERSION_HELP),

atlas_init/cli_helper/go.py

@@ -1,34 +1,214 @@
 import logging
 import os
+from concurrent.futures import ThreadPoolExecutor, wait
+from enum import StrEnum
 from pathlib import Path
 
-from atlas_init.cli_helper.run import run_command_is_ok
+from model_lib import Entity
+from pydantic import Field
+
+from atlas_init.cli_helper.run import run_command_is_ok_output
+from atlas_init.cli_tf.go_test_run import (
+    GoTestContext,
+    GoTestContextStep,
+    GoTestRun,
+    parse,
+)
 from atlas_init.settings.config import TestSuite
 from atlas_init.settings.env_vars import AtlasInitSettings
+from atlas_init.settings.path import DEFAULT_DOWNLOADS_DIR
 
 logger = logging.getLogger(__name__)
 
 
+class GoTestMode(StrEnum):
+    package = "package"
+    individual = "individual"
+
+
+class GoEnvVars(StrEnum):
+    manual = "manual"
+    vscode = "vscode"
+
+
+class GoTestResult(Entity):
+    runs: dict[str, list[GoTestRun]] = Field(default_factory=dict)
+    failure_names: set[str] = Field(default_factory=set)
+
+    test_name_package_path: dict[str, Path] = Field(default_factory=dict)
+
+    def add_test_package_path(self, test_name: str, package_path: Path):
+        if old_path := self.test_name_package_path.get(test_name):
+            logger.warning(f"overwriting test_name={test_name} with package_path={old_path} --> {package_path}")
+        self.test_name_package_path[test_name] = package_path
+
+    def add_test_results_all_pass(self, test_name: str, test_results: list[GoTestRun]) -> bool:
+        prev_test_results = self.runs.setdefault(test_name, [])
+        if prev_test_results:
+            logger.warning(f"2nd time test results for {test_name}")
+        for result in test_results:
+            log_path = _log_path(test_name)
+            result.log_path = log_path
+        prev_test_results.extend(test_results)
+        return all(run.is_pass for run in test_results)
+
+
 def run_go_tests(
     repo_path: Path,
     repo_alias: str,
     package_prefix: str,
     settings: AtlasInitSettings,
     groups: list[TestSuite],
-):
-    extra_vars = settings.load_env_vars(settings.env_vars_vs_code)
-    logger.info(f"go test env-vars-extra: {sorted(extra_vars)}")
-    test_env = os.environ | extra_vars
-    ci_value = test_env.pop("CI", None)
-    if ci_value:
+    mode: GoTestMode = GoTestMode.package,
+    *,
+    dry_run: bool = False,
+    timeout_minutes: int = 300,
+    concurrent_runs: int = 20,
+    re_run: bool = False,
+    env_vars: GoEnvVars = GoEnvVars.vscode,
+) -> GoTestResult:
+    test_env = _resolve_env_vars(settings, env_vars)
+    if ci_value := test_env.pop("CI", None):
         logger.warning(f"pooped CI={ci_value}")
+    results = GoTestResult()
+    commands_to_run: dict[str, str] = {}
     for group in groups:
-        packages = ",".join(f"{package_prefix}/{pkg}" for pkg in group.repo_go_packages.get(repo_alias, []))
+        package_paths = group.repo_go_packages.get(repo_alias, [])
+        packages = ",".join(f"{package_prefix}/{pkg}" for pkg in package_paths)
         if not packages:
             logger.warning(f"no go packages for suite: {group}")
             continue
-        command = f"go test {packages} -v -run ^TestAcc* -timeout 300m".split(" ")
-        if not group.sequential_tests:
-            command.extend(["-parallel", "20"])
-        is_ok = run_command_is_ok(command, test_env, cwd=repo_path, logger=logger)
-        assert is_ok, f"go tests failed for {group}"
+        if mode == GoTestMode.individual:
+            test_names = find_individual_tests(repo_path, package_paths)
+            for name, pkg_path in test_names.items():
+                results.add_test_package_path(name, pkg_path)
+                commands_to_run[name] = f"go test {packages} -v -run ^{name}$ -timeout {timeout_minutes}m"
+        elif mode == GoTestMode.package:
+            command = f"go test {packages} -v -run ^TestAcc* -timeout {timeout_minutes}m"
+            if not group.sequential_tests:
+                command = f"{command} -parallel {concurrent_runs}"
+            commands_to_run[group.name] = command
+        else:
+            raise NotImplementedError(f"mode={mode}")
+    commands_str = "\n".join(f"'{name}': '{command}'" for name, command in sorted(commands_to_run.items()))
+    logger.info(f"will run the following commands:\n{commands_str}")
+    if dry_run:
+        return results
+    if not commands_to_run:
+        logger.warning("no tests to run!")
+        return results
+    return _run_tests(
+        results,
+        repo_path,
+        commands_to_run,
+        test_env,
+        test_timeout_s=timeout_minutes * 60,
+        max_workers=concurrent_runs,
+        re_run=re_run,
+    )
+
+
+def _resolve_env_vars(settings: AtlasInitSettings, env_vars: GoEnvVars) -> dict[str, str]:
+    if env_vars == GoEnvVars.manual:
+        extra_vars = settings.load_profile_manual_env_vars(skip_os_update=True)
+    elif env_vars == GoEnvVars.vscode:
+        extra_vars = settings.load_env_vars(settings.env_vars_vs_code)
+    else:
+        raise NotImplementedError(f"don't know how to load env_vars={env_vars}")
+    test_env = os.environ | extra_vars | {"TF_ACC": "1", "TF_LOG": "DEBUG"}
+    logger.info(f"go test env-vars-extra: {sorted(extra_vars)}")
+    return test_env
+
+
+def find_individual_tests(repo_path: Path, package_paths: list[str]) -> dict[str, Path]:
+    tests = {}
+    for package_path in package_paths:
+        package_abs_path = repo_path / package_path.lstrip(".").lstrip("/")
+        for go_file in package_abs_path.glob("*.go"):
+            with go_file.open() as f:
+                for line in f:
+                    if line.startswith("func TestAcc"):
+                        test_name = line.split("(")[0].strip().removeprefix("func ")
+                        tests[test_name] = package_abs_path
+    return tests
+
+
+def _run_tests(
+    results: GoTestResult,
+    repo_path: Path,
+    commands_to_run: dict[str, str],
+    test_env: dict[str, str],
+    test_timeout_s: int = 301 * 60,
+    max_workers: int = 2,
+    *,
+    re_run: bool = False,
+) -> GoTestResult:
+    futures = {}
+    actual_workers = min(max_workers, len(commands_to_run)) or 1
+    with ThreadPoolExecutor(max_workers=actual_workers) as pool:
+        for name, command in sorted(commands_to_run.items()):
+            log_path = _log_path(name)
+            if log_path.exists() and log_path.read_text() and not re_run:
+                logger.info(f"skipping {name} because log exists")
+                continue
+            command_env = {**test_env, "TF_LOG_PATH": str(log_path)}
+            future = pool.submit(
+                run_command_is_ok_output,
+                command=command,
+                env=command_env,
+                cwd=repo_path,
+                logger=logger,
+            )
+            futures[future] = name
+        done, not_done = wait(futures.keys(), timeout=test_timeout_s)
+        for f in not_done:
+            logger.warning(f"timeout to run command name = {futures[f]}")
+    for f in done:
+        name: str = futures[f]
+        try:
+            ok, command_out = f.result()
+        except Exception:
+            logger.exception(f"failed to run command for {name}")
+            results.failure_names.add(name)
+            continue
+        context = GoTestContext(
+            name=name,
+            html_url=f"file://{_log_path(name)}",
+            steps=[GoTestContextStep(name="local-run")],
+        )
+        try:
+            parsed_tests = list(parse(command_out.splitlines(), context, test_step_nr=0))
+        except Exception:
+            logger.exception(f"failed to parse tests for {name}")
+            results.failure_names.add(name)
+            continue
+        if not parsed_tests and not ok:
+            results.failure_names.add(name)
+            logger.error(f"failed to run tests for {name}: {command_out}")
+            continue
+        if not parsed_tests:
+            logger.warning(f"failed to parse tests for {name}: {command_out}")
+            continue
+        if not ok:
+            logger.warning(f"failing tests for {name}: {command_out}")
+        if not results.add_test_results_all_pass(name, parsed_tests):
+            results.failure_names.add(name)
+    if failure_names := results.failure_names:
+        move_failed_logs_to_error_dir(failure_names)
+        logger.error(f"failed to run tests: {sorted(failure_names)}")
+    return results
+
+
+def move_failed_logs_to_error_dir(failures: set[str]):
+    error_dir = DEFAULT_DOWNLOADS_DIR / "failures"
+    for log in DEFAULT_DOWNLOADS_DIR.glob("*.log"):
+        if log.stem in failures:
+            text = log.read_text()
+            assert "\n" in text
+            first_line = text.split("\n", maxsplit=1)[0]
+            ts = first_line.split(" ")[0]
+            log.rename(error_dir / f"{ts}.{log.name}")
+
+
+def _log_path(name: str) -> Path:
+    return DEFAULT_DOWNLOADS_DIR / f"{name}.log"

atlas_init/cli_helper/run.py

@@ -8,6 +8,7 @@ from tempfile import TemporaryDirectory
 from typing import IO, TypeVar
 
 import typer
+from zero_3rdparty.id_creator import simple_id
 
 StrT = TypeVar("StrT", bound=str)
 
@@ -94,6 +95,15 @@ def run_command_receive_result(
     return output_text
 
 
+def run_command_is_ok_output(command: str, cwd: Path, logger: Logger, env: dict | None = None) -> tuple[bool, str]:
+    with TemporaryDirectory() as temp_dir:
+        result_file = Path(temp_dir) / f"{simple_id()}.txt"
+        with open(result_file, "w") as file:
+            is_ok = run_command_is_ok(command.split(), env=env, cwd=cwd, logger=logger, output=file)
+        output_text = result_file.read_text().strip()
+    return is_ok, output_text
+
+
 def add_to_clipboard(clipboard_content: str, logger: Logger):
     if pb_binary := find_binary_on_path("pbcopy", logger, allow_missing=True):
         subprocess.run(pb_binary, text=True, input=clipboard_content, check=True)

atlas_init/cli_root/go_test.py

@@ -0,0 +1,88 @@
+import logging
+
+import typer
+
+from atlas_init.cli_helper.go import GoEnvVars, GoTestMode, GoTestResult, run_go_tests
+from atlas_init.cli_tf.mock_tf_log import MockTFLog, mock_tf_log, resolve_admin_api_path
+from atlas_init.repos.path import Repo, current_repo, current_repo_path
+from atlas_init.settings.env_vars import active_suites, init_settings
+from atlas_init.typer_app import app_command
+
+logger = logging.getLogger(__name__)
+
+
+@app_command()
+def go_test(
+    mode: GoTestMode = typer.Option("package", "-m", "--mode", help="package|individual"),
+    dry_run: bool = typer.Option(False, help="only log out the commands to be run"),
+    timeout_minutes: int = typer.Option(300, "-t", "--timeout", help="timeout in minutes"),
+    concurrent_runs: int = typer.Option(20, "-c", "--concurrent", help="number of concurrent runs"),
+    re_run: bool = typer.Option(False, "-r", "--re-run", help="re-run the tests if the log already exist"),
+    export_mock_tf_log: bool = typer.Option(False, "-e", "--export", help="export the mock-tf-log"),
+    env_method: GoEnvVars = typer.Option(GoEnvVars.manual, "--env", help="|".join(list(GoEnvVars))),
+):
+    if export_mock_tf_log and mode != GoTestMode.individual:
+        err_msg = "exporting mock-tf-log is only supported for individual tests"
+        raise ValueError(err_msg)
+    settings = init_settings()
+    suites = active_suites(settings)
+    sorted_suites = sorted(suite.name for suite in suites)
+    logger.info(f"running go tests for {len(suites)} test-suites: {sorted_suites}")
+    results: GoTestResult | None = None
+    match repo_alias := current_repo():
+        case Repo.CFN:
+            raise NotImplementedError
+        case Repo.TF:
+            repo_path = current_repo_path()
+            package_prefix = settings.config.go_package_prefix(repo_alias)
+            results = run_go_tests(
+                repo_path,
+                repo_alias,
+                package_prefix,
+                settings,
+                suites,
+                mode,
+                dry_run=dry_run,
+                timeout_minutes=timeout_minutes,
+                concurrent_runs=concurrent_runs,
+                re_run=re_run,
+                env_vars=env_method,
+            )
+        case _:
+            raise NotImplementedError
+    if results is None:
+        error_msg = "no results found"
+        raise ValueError(error_msg)
+    if export_mock_tf_log:
+        _export_mock_tf_logs(results)
+    # use the test_results: dict[str, list[GoTestRun]]
+    # TODO: create_detailed_summary()
+
+
+def _export_mock_tf_logs(results: GoTestResult):
+    package_paths = results.test_name_package_path
+    admin_api_path = resolve_admin_api_path("", sdk_branch="main", admin_api_path="")
+    for test_name, runs in results.runs.items():
+        package_path = package_paths.get(test_name)
+        if package_path is None:
+            logger.warning(f"no package path found for test_name={test_name}")
+            continue
+        assert len(runs) == 1, f"expected only 1 run for test_name={test_name}, got {len(runs)}"
+        run = runs[0]
+        tpf_package_path = package_path.with_name(f"{package_path.name}tpf") / "testdata"
+        default_package_path = package_path / "testdata"
+        if not tpf_package_path.exists():
+            logger.warning(
+                f"tpf_package_path={tpf_package_path} doesn't exist, adding mocked data to {default_package_path}"
+            )
+            tpf_package_path = default_package_path
+        tf_log_path = run.log_path
+        assert tf_log_path, f"test didn't set tf_log_path: {test_name}"
+        req = MockTFLog(
+            log_path=tf_log_path,
+            output_dir=tpf_package_path,
+            admin_api_path=admin_api_path,
+            package_name=package_path.name,
+        )
+        mocked_yaml = mock_tf_log(req)
+        logger.info(f"mocked TestConfig saved to {mocked_yaml}")

atlas_init/cli_tf/debug_logs.py

@@ -11,13 +11,15 @@ from pydantic import ValidationError, model_validator
 logger = logging.getLogger(__name__)
 
 
-def parsed(payload: str) -> tuple[dict[str, Any], list]:
+def parsed(payload: str) -> tuple[dict[str, Any] | None, list | None, bool | None]:
     with suppress(ValueError):
         resp = json.loads(payload)
         if isinstance(resp, dict):
-            return resp, []
+            return resp, None, None
         if isinstance(resp, list):
-            return {}, resp
+            return None, resp, None
+    if payload.strip() in {"true", "false"}:
+        return None, None, payload.strip() == "true"
     raise ValueError(f"Could not parse payload: {payload}")
 
 
@@ -98,19 +100,19 @@ class SDKRoundtrip(Entity):
     @property
     def version(self) -> str:
         content_type = self.response.headers.get("Content-Type", "v1")
-        try:
+        content_type_req = self.request.headers.get("Accept", "v1")
+        with suppress(ValueError):
             return extract_version(content_type)
-        except ValueError:
-            logger.warning(f"failed to extract version from response header ({content_type}), trying request")
-        content_type = self.request.headers.get("Accept", "v1")
-        return extract_version(content_type)
+        with suppress(ValueError):
+            return extract_version(content_type_req)
+        raise ValueError(f"Could not extract version from req/resp: {content_type} or {content_type_req}")
 
     @model_validator(mode="after")
     def ensure_match(self) -> Self:
         req = self.request
         resp = self.response
-        _, resp_payload_list = parsed(resp.text)
-        if req.expect_list_response and not resp_payload_list:
+        _, resp_payload_list, __ = parsed(resp.text)
+        if req.expect_list_response and resp_payload_list is None:
             raise ValueError(f"Expected list response but got dict: {resp.text}")
         return self
 
@@ -159,7 +161,7 @@ def parse_http_requests(logs: str) -> list[SDKRoundtrip]:
     Can say that expected payload is either a list or a dict and if it ends with an identifier it is higher chance for a dict
     """
     test_name = parse_test_name(logs)
-    logger.info(f"Finding http requests for test name: {test_name}")
+    logger.info(f"Finding http requests for test name: '{test_name}'")
     requests, responses = parse_raw_req_responses(logs)
     tf_step_starts = [i for i, line in enumerate(logs.splitlines()) if MARKER_START_STEP in line]
     used_responses: set[int] = set()

atlas_init/cli_tf/debug_logs_test_data.py

@@ -31,7 +31,7 @@ class RequestInfo(Entity):
 
     @property
     def id(self):
-        return "__".join((self.method, self.path, self.version, self.text))  # noqa: FLY002
+        return "__".join(part for part in (self.method, self.path, self.version, self.text) if part)
 
 
 class StepRequests(Entity):
@@ -143,9 +143,9 @@ class MockRequestData(Entity):
     def replace_text_variables(self):
         for step in self.steps:
             for request in step.all_requests:
-                request.text = normalize_text(request.text, self.variables)
+                request.text = normalize_text(request.text, self.variables, expect_json=True)
                 for response in request.responses:
-                    response.text = normalize_text(response.text, self.variables)
+                    response.text = normalize_text(response.text, self.variables, expect_json=True)
 
     def prune_duplicate_responses(self):
         for step in self.steps:
@@ -202,10 +202,10 @@ def find_normalized_path(path: str, api_spec_paths: list[ApiSpecPath]) -> ApiSpe
     raise ValueError(f"Could not find path: {path}")
 
 
-def normalize_text(text: str, variables: dict[str, str]) -> str:
+def normalize_text(text: str, variables: dict[str, str], *, expect_json: bool = False) -> str:
     for var, value in variables.items():
         text = text.replace(value, f"{{{var}}}")
-    if not text:
+    if not text or not expect_json:
         return text
     try:
         parsed_text = json.loads(text)
@@ -260,8 +260,8 @@ def create_mock_data(
         for modifier in modifiers:
             if modifier.match(rt, normalized_path):
                 modifier.modification(rt)
-        normalized_text = normalize_text(rt.request.text, mock_data.variables)
-        normalized_response_text = normalize_text(rt.response.text, mock_data.variables)
+        normalized_text = normalize_text(rt.request.text, mock_data.variables, expect_json=True)
+        normalized_response_text = normalize_text(rt.response.text, mock_data.variables, expect_json=True)
         mock_data.add_roundtrip(rt, normalized_path, normalized_text, normalized_response_text, is_diff(rt))
     mock_data.replace_text_variables()
     if prune_duplicates:

atlas_init/cli_tf/debug_logs_test_data_package_config.py

@@ -0,0 +1,48 @@
+import json
+import logging
+
+from atlas_init.cli_tf.debug_logs import SDKRoundtrip, parsed
+from atlas_init.cli_tf.debug_logs_test_data import RTModifier
+
+logger = logging.getLogger(__name__)
+
+
+def add_label_tags(rt: SDKRoundtrip):
+    logger.info(f"Adding labels and tags to {rt.id}")
+    request = rt.request
+    req_dict, req_list, req_bool = parsed(request.text)
+    response = rt.response
+    resp_dict, resp_list, req_bool = parsed(response.text)
+    if resp_list or req_list or req_bool is not None:
+        return
+    resp_dict = resp_dict or {}
+    req_dict = req_dict or {}
+    for extra_field in ["labels", "tags"]:
+        if extra_field not in resp_dict:
+            resp_dict[extra_field] = []
+        if extra_field not in req_dict:
+            req_dict[extra_field] = []
+    request.text = json.dumps(req_dict, indent=1, sort_keys=True)
+    response.text = json.dumps(resp_dict, indent=1, sort_keys=True)
+
+
+cluster_modifier = RTModifier(
+    version="2024-08-05",
+    method="POST",
+    path="/api/atlas/v2/groups/{groupId}/clusters",
+    modification=add_label_tags,
+)
+
+
+def package_modifiers(pkg_name: str) -> list[RTModifier]:
+    # sourcery skip: assign-if-exp, reintroduce-else
+    if pkg_name == "advancedcluster":
+        return [cluster_modifier]
+    return []
+
+
+def package_skip_suffixes(pkg_name: str) -> list[str]:
+    # sourcery skip: assign-if-exp, reintroduce-else
+    if pkg_name == "resourcepolicy":
+        return [":validate"]
+    return []

atlas_init/cli_tf/go_test_run.py

@@ -5,11 +5,13 @@ import re
 from collections.abc import Iterable
 from enum import StrEnum
 from functools import total_ordering
+from pathlib import Path
 
 import humanize
 from github.WorkflowJob import WorkflowJob
 from model_lib import Entity, Event, utc_datetime
-from pydantic import Field
+from pydantic import Field, field_validator
+from zero_3rdparty.datetime_utils import utc_now
 
 logger = logging.getLogger(__name__)
 
@@ -35,6 +37,24 @@ class LineInfo(Event):
     text: str
 
 
+class GoTestContextStep(Entity):
+    name: str
+
+
+class GoTestContext(Entity):
+    """Abstraction on WorkflowJob to also support local runs"""
+
+    name: str
+    created_at: utc_datetime = Field(default_factory=utc_now)
+    steps: list[GoTestContextStep] = Field(default_factory=list)
+    html_url: str = "http://localhost"
+
+    @classmethod
+    def from_local_run(cls, name: str, steps: list[GoTestContextStep]) -> GoTestContext:
+        raise NotImplementedError
+        # return cls(name=name, steps=steps)
+
+
 @total_ordering
 class GoTestRun(Entity):
     name: str
@@ -42,8 +62,9 @@ class GoTestRun(Entity):
     start_line: LineInfo
     ts: utc_datetime
     finish_ts: utc_datetime | None = None
-    job: WorkflowJob
+    job: GoTestContext | WorkflowJob
     test_step: int
+    log_path: Path | None = None
 
     finish_line: LineInfo | None = None
     context_lines: list[str] = Field(default_factory=list)
@@ -87,6 +108,10 @@ class GoTestRun(Entity):
     def is_failure(self) -> bool:
         return self.status == GoTestStatus.FAIL
 
+    @property
+    def is_pass(self) -> bool:
+        return self.status == GoTestStatus.PASS
+
     def add_line_match(self, match: LineMatch, line: str, line_number: int) -> None:
         self.run_seconds = match.run_seconds or self.run_seconds
         self.finish_line = LineInfo(number=line_number, text=line)
@@ -99,7 +124,7 @@ class GoTestRun(Entity):
         match: LineMatch,
         line: str,
         line_number: int,
-        job: WorkflowJob,
+        job: WorkflowJob | GoTestContext,
         test_step_nr: int,
     ) -> GoTestRun:
         start_line = LineInfo(number=line_number, text=line)
@@ -115,15 +140,20 @@ class GoTestRun(Entity):
 
 
 class LineMatch(Event):
-    ts: utc_datetime
+    ts: utc_datetime = Field(default_factory=utc_now)
     status: GoTestStatus
     name: str
     run_seconds: float | None = None
 
+    @field_validator("ts", mode="before")
+    @classmethod
+    def remove_none(cls, v):
+        return v or utc_now()
+
 
 _status_options = "|".join(list(GoTestStatus))
 line_result = re.compile(
-    r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z?)\s[-=]+\s"
+    r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z?)?\s?[-=]+\s"
     + r"(?P<status>%s):?\s+" % _status_options  # noqa: UP031
     + r"(?P<name>[\w_]+)"
     + r"\s*\(?(?P<run_seconds>[\d\.]+)?s?\)?"
@@ -140,13 +170,13 @@ def match_line(line: str) -> LineMatch | None:
     2024-06-26T04:41:47.7228652Z --- PASS: TestAccNetworkRSPrivateLinkEndpointGCP_basic (424.50s)
     """
     if match := line_result.match(line):
-        line_match = LineMatch(**match.groupdict())
+        line_match = LineMatch(**match.groupdict())  # type: ignore
         return None if _test_name_is_nested(line_match.name, line) else line_match
     return None
 
 
 context_start_pattern = re.compile(
-    r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z?)\s[-=]+\s" r"NAME\s+" r"(?P<name>[\w_]+)"
+    r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z?)?\s?[-=]+\s" r"NAME\s+" r"(?P<name>[\w_]+)"
 )
 
 
@@ -157,7 +187,7 @@ def context_start_match(line: str) -> str:
 
 
 context_line_pattern = re.compile(
-    r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z?)" r"\s{5}" r"(?P<indent>\s*)" r"(?P<relevant_line>.*)"
+    r"(?P<ts>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z?)?" r"\s{5}" r"(?P<indent>\s*)" r"(?P<relevant_line>.*)"
 )
 
 
@@ -168,7 +198,7 @@ def extract_context(line: str) -> str:
     return ""
 
 
-def parse(test_lines: list[str], job: WorkflowJob, test_step_nr: int) -> Iterable[GoTestRun]:
+def parse(test_lines: list[str], job: WorkflowJob | GoTestContext, test_step_nr: int) -> Iterable[GoTestRun]:
     tests: dict[str, GoTestRun] = {}
     context_lines: list[str] = []
     current_context_test = ""

atlas_init/cli_tf/mock_tf_log.py

@@ -1,11 +1,14 @@
 import json
 import logging
 import time
+from collections.abc import Callable
+from io import StringIO
 from pathlib import Path
 from typing import Self
 
 import typer
-from model_lib import Entity, dump
+import yaml
+from model_lib import Entity
 from pydantic import Field, model_validator
 from zero_3rdparty import file_utils
 
@@ -15,8 +18,17 @@ from atlas_init.cli_tf.debug_logs import (
     parse_http_requests,
     parse_test_name,
 )
-from atlas_init.cli_tf.debug_logs_test_data import create_mock_data, default_is_diff
-from atlas_init.repos.go_sdk import api_spec_path_transformed, download_admin_api, parse_api_spec_paths
+from atlas_init.cli_tf.debug_logs_test_data import (
+    RTModifier,
+    create_mock_data,
+    default_is_diff,
+)
+from atlas_init.cli_tf.debug_logs_test_data_package_config import package_modifiers, package_skip_suffixes
+from atlas_init.repos.go_sdk import (
+    api_spec_path_transformed,
+    download_admin_api,
+    parse_api_spec_paths,
+)
 from atlas_init.settings.path import DEFAULT_DOWNLOADS_DIR
 
 logger = logging.getLogger(__name__)
@@ -28,6 +40,10 @@ class MockTFLog(Entity):
     admin_api_path: Path
     diff_skip_suffixes: list[str] = Field(default_factory=list)
     keep_duplicates: bool = False
+    modifiers: list[RTModifier] = Field(default_factory=list)
+    package_name: str = ""
+    log_diff_roundtrips: bool = False
+    skip_default_package_config: bool = False
 
     @model_validator(mode="after")
     def ensure_paths_exist(self) -> Self:
@@ -38,28 +54,49 @@ class MockTFLog(Entity):
         if not self.output_dir.exists():
             raise ValueError(f"output_dir: '{self.output_dir}' doesn't exist")
         assert self.output_dir.name == "testdata", "output_path should be a directory named testdata"
+        if (package_name := self.package_name) and not self.skip_default_package_config:
+            self.modifiers.extend(package_modifiers(package_name))
+            self.diff_skip_suffixes.extend(package_skip_suffixes(package_name))
         return self
 
     def differ(self, rt: SDKRoundtrip) -> bool:
         return default_is_diff(rt) and not any(rt.request.path.endswith(suffix) for suffix in self.diff_skip_suffixes)
 
 
-def mock_tf_log(req: MockTFLog) -> None:
+def mock_tf_log(req: MockTFLog) -> Path:
     log_file_text = req.log_path.read_text()
     test_name = parse_test_name(log_file_text)
     roundtrips = parse_http_requests(log_file_text)
+    logger.info(f"Found #{len(roundtrips)} roundtrips")
+    if req.log_diff_roundtrips:
+        log_diff_roundtrips(roundtrips, req.differ)
     api_spec_paths = parse_api_spec_paths(req.admin_api_path)
     data = create_mock_data(
         roundtrips,
         api_spec_paths,
         is_diff=req.differ,
         prune_duplicates=not req.keep_duplicates,
+        modifiers=req.modifiers,
     )
     # avoid anchors
-    data_yaml = dump(json.loads(dump(data, "json")), "yaml")
+    data_json = data.model_dump_json(exclude_none=True)
+    data_parsed = json.loads(data_json)
+    s = StringIO()
+    yaml.safe_dump(
+        data_parsed,
+        s,
+        default_flow_style=False,
+        width=100_000,
+        allow_unicode=True,
+        sort_keys=False,
+    )
+    data_yaml = s.getvalue()
+    test_name = test_name.replace("TestAcc", "TestMock")
     output_path = req.output_dir / f"{test_name}.yaml"
+    logger.info(f"Variables found {data.variables}")
     logger.info(f"Writing to {output_path}")
     file_utils.ensure_parents_write_text(output_path, data_yaml)
+    return output_path
 
 
 def mock_tf_log_cmd(
@@ -68,7 +105,7 @@ def mock_tf_log_cmd(
         "",
         "-o",
         "--output-testdir",
-        help="the path to the output test directory, for example: internal/service/advancedclustertpf/testdata/, uses cwd/testdata by default",
+        help="the path to the output test directory, for example: internal/service/advancedclustertpf/testdata/, uses $(cwd)/testdata by default",
     ),
     sdk_repo_path_str: str = option_sdk_repo_path,
     sdk_branch: str = typer.Option("main", "-b", "--branch", help="the branch for downloading openapi spec"),
@@ -77,6 +114,9 @@ def mock_tf_log_cmd(
     ),
     diff_skip_suffixes: list[str] = typer.Option(..., "-s", "--skip-suffixes", default_factory=list),
     keep_duplicates: bool = typer.Option(False, "-keep", "--keep-duplicates", help="keep duplicate requests"),
+    log_diff_roundtrips: bool = typer.Option(
+        False, "-l", "--log-diff-roundtrips", help="print out the roundtrips used in diffs"
+    ),
 ):
     cwd = Path.cwd()
     default_testdir = cwd / "testdata"
@@ -87,6 +127,7 @@ def mock_tf_log_cmd(
         admin_api_path=resolved_admin_api_path,
         diff_skip_suffixes=diff_skip_suffixes,
         keep_duplicates=keep_duplicates,
+        log_diff_roundtrips=log_diff_roundtrips,
     )
     mock_tf_log(event_in)
 
@@ -116,3 +157,20 @@ def resolve_admin_api_path(sdk_repo_path_str: str, sdk_branch: str, admin_api_pa
     assert resolved_admin_api_path.exists(), f"unable to resolve admin_api_path={resolved_admin_api_path}"
     assert resolved_admin_api_path.is_file(), f"not a file admin_api_path={resolved_admin_api_path}"
     return resolved_admin_api_path
+
+
+def log_diff_roundtrips(roundtrips: list[SDKRoundtrip], differ: Callable[[SDKRoundtrip], bool] | None = None):
+    differ = differ or default_is_diff
+    diff_count = 0
+    step_nr = 0
+    for rt in roundtrips:
+        if not differ(rt):
+            continue
+        if rt.step_number != step_nr:
+            logger.info(f"{'-' * 80}\nStep {rt.step_number}")
+            step_nr = rt.step_number
+        diff_count += 1
+        logger.info(
+            f"\n{rt.request.method} {rt.request.path}\n{rt.request.text}\n{rt.response.status}-{rt.response.status_text}\n{rt.response.text}"
+        )
+    logger.info(f"Diffable requests: {diff_count}")

atlas_init/settings/env_vars.py

@@ -170,9 +170,12 @@ class AtlasInitPaths(BaseSettings):
         assert env_path.exists(), f"no env-vars exist {env_path} have you forgotten apply?"
         return load_dotenv(env_path)
 
-    def load_profile_manual_env_vars(self) -> dict[str, str]:
+    def load_profile_manual_env_vars(self, *, skip_os_update: bool = False) -> dict[str, str]:
+        # sourcery skip: dict-assign-update-to-union
         manual_env_vars = self.manual_env_vars
         if manual_env_vars:
+            if skip_os_update:
+                return manual_env_vars
             logger.warning(f"loading manual env-vars from {self.env_file_manual}")
             os.environ.update(manual_env_vars)
         else:

atlas_init/typer_app.py

@@ -27,9 +27,10 @@ app_command = partial(
 
 
 def extra_root_commands():
-    from atlas_init.cli_root import trigger
+    from atlas_init.cli_root import go_test, trigger
 
     assert trigger
+    assert go_test
 
 
 @app.callback(invoke_without_command=True)

{atlas_init-0.3.1.dist-info → atlas_init-0.3.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: atlas-init
-Version: 0.3.1
+Version: 0.3.3
 Project-URL: Documentation, https://github.com/EspenAlbert/atlas-init#readme
 Project-URL: Issues, https://github.com/EspenAlbert/atlas-init/issues
 Project-URL: Source, https://github.com/EspenAlbert/atlas-init

{atlas_init-0.3.1.dist-info → atlas_init-0.3.3.dist-info}/RECORD

@@ -1,11 +1,11 @@
-atlas_init/__init__.py,sha256=Mja1wzBho0kdXUYEG2JEXu72cDv6N_Uf0xLaUN7aQsA,372
+atlas_init/__init__.py,sha256=x8cFU0CpX_K9Ha_EhynKLP9QaJvcEH7phPh9svYKiZI,372
 atlas_init/__main__.py,sha256=dY1dWWvwxRZMmnOFla6RSfti-hMeLeKdoXP7SVYqMUc,52
-atlas_init/atlas_init.yaml,sha256=GMyJVhKKRc7WzEu7fafmWgeTsDaExTLv7QvXOmE_Brg,1907
-atlas_init/cli.py,sha256=IiOEC_Jry6vrSDH3_OvsU50F-_3iVIS4tV6-R7659fY,9642
+atlas_init/atlas_init.yaml,sha256=6UC10VjXpn80RMtn-LsuJ-Ghoql5KBWKgzs3BixHfIo,2040
+atlas_init/cli.py,sha256=xOnAOUccHDLkivICdF0GsLhccr_IxvnTKTbe1KGW7kU,8971
 atlas_init/cli_args.py,sha256=tiwUYAE0JBSl9lHV6VJ41vFCU90ChBZ4mKvi-YoF_HY,541
 atlas_init/humps.py,sha256=l0ZXXuI34wwd9TskXhCjULfGbUyK-qNmiyC6_2ow6kU,7339
 atlas_init/terraform.yaml,sha256=qPrnbzBEP-JAQVkYadHsggRnDmshrOJyiv0ckyZCxwY,2734
-atlas_init/typer_app.py,sha256=zbvYUlZrF4TZEPEwpa33fVSLVKcxRamuXCgF1FCUhCU,2068
+atlas_init/typer_app.py,sha256=nJbSuaAVhVzbxqNup1m5Nmszl2X3FOtZyI7Rjmz4JSs,2096
 atlas_init/cli_cfn/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/cli_cfn/app.py,sha256=iMukpUDgsAgZh_U_APGZB3gmewOo-3vtFK0byJuDz9w,6649
 atlas_init/cli_cfn/aws.py,sha256=GbohR7uczSGwQjLEYozCmlxbeIHo1uwQIJMwsh7kF7M,17894
@@ -13,23 +13,25 @@ atlas_init/cli_cfn/cfn_parameter_finder.py,sha256=tAadNF1M_U2BTY-m9fXVXFXNQRvfud
 atlas_init/cli_cfn/example.py,sha256=pQNpFreuv58O3lanLy5Kunp8GxG8i9PWjuWsYlpv2tg,8320
 atlas_init/cli_cfn/files.py,sha256=vjop9G8rGMgyRe4fX5eWNX5H-YGAmk-fNUqUGErI7xg,1720
 atlas_init/cli_helper/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-atlas_init/cli_helper/go.py,sha256=LVMXTWvEYOHOul_UG5uQT3QAoL0H1f1O7e4d-11kPrw,1232
-atlas_init/cli_helper/run.py,sha256=LvzOXFBotGZQ7KZjivr4e7Aq8AUnxy7vGpH1rluSAN8,2988
+atlas_init/cli_helper/go.py,sha256=ulRjwQ9JVXAd0FUgZJ1gPzeINx0B9FH9jUZ1O7Skilk,8033
+atlas_init/cli_helper/run.py,sha256=njE_ua8x_glOo6eOGa4NgZqpLcqOo3eALydrZ0bCXW4,3486
 atlas_init/cli_helper/sdk.py,sha256=exh58-VZwxtosaxM269C62EEy1VnpJPOVziPDPkGsmE,2983
 atlas_init/cli_helper/sdk_auto_changes.py,sha256=oWyXw7P0PdO28hclRvza_RcIVXAyzu0lCYTJTNBDMeo,189
 atlas_init/cli_helper/tf_runner.py,sha256=OYdC-Y6i-xRh8_LCudKdtP7CEYEO9e67nVhholN29eg,3636
 atlas_init/cli_root/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+atlas_init/cli_root/go_test.py,sha256=K8N-EIgry21ILYGJqj5Y7kjPXPl4UTOS9WDk3m3nhjY,3878
 atlas_init/cli_root/trigger.py,sha256=oEgqb_l25tyYgUaFHEuChcOCJA7k3mnRa4D-Myz-Igs,5789
 atlas_init/cli_tf/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/cli_tf/app.py,sha256=0Y5c-Pc9ibOz6kXvFlL-yhH_fx1nHLgBgK9OAVqjX9s,11390
 atlas_init/cli_tf/changelog.py,sha256=biWYKf1pZvXZ-jEgcZ5q9sY7nTGrL2PuI0h9mCILf_g,3181
-atlas_init/cli_tf/debug_logs.py,sha256=lnB5BpcEooVzGd2RLxbwAVQs0ZYXzRKy5sHa0hftHI8,8799
-atlas_init/cli_tf/debug_logs_test_data.py,sha256=bv4gqhHSNEnQqIijrcjvEUA0M6S-aeo73V4mji0pKCM,9435
+atlas_init/cli_tf/debug_logs.py,sha256=q71ZNOnQOz1ikPCyqUz_6zyd4Bm1QVkCkTcixJBZ1xI,8988
+atlas_init/cli_tf/debug_logs_test_data.py,sha256=G4pnuWJ7PAQd3NXRKAtwAPC6Ne-PgpzaTZHQ9waqxZI,9565
+atlas_init/cli_tf/debug_logs_test_data_package_config.py,sha256=0GB-m8l9TWL4vstnFVO2jw5Jvtlz9WfHTp-9RmaPugw,1473
 atlas_init/cli_tf/github_logs.py,sha256=VD7qhlXNuG21eTuJ5VI7rsflp5WHSodfngkRVgQlumw,8114
-atlas_init/cli_tf/go_test_run.py,sha256=ZoQSvIasmWauFxZJrWL0ObFX-P0k-D3c_ep3OnPY4zs,5842
+atlas_init/cli_tf/go_test_run.py,sha256=LQUQ-3zJ8EUCixwu33QTAzUns3um793osst8tE0UKjk,6792
 atlas_init/cli_tf/go_test_run_format.py,sha256=OUd6QPHDeTzbwVuh6MhP-xXgjOOGP9W_sCLJ8KylBTs,1201
 atlas_init/cli_tf/go_test_summary.py,sha256=agr4SITgxchjgOzRpScoTUk-iG38QDLkpnsMtTW9GTY,5382
-atlas_init/cli_tf/mock_tf_log.py,sha256=c0geBR74UkHiyElnV0R_yTuXUgP4F_H53rbGj6D99yc,4958
+atlas_init/cli_tf/mock_tf_log.py,sha256=311sUVpxbvcz6Qdpz2Z1kFQm67zfjs4aUsQOKrJ2LrY,6988
 atlas_init/cli_tf/schema.py,sha256=iwvb4wD2Wba0MMu7ooTNAIi1jHbpLiXGPOT51_o_YW8,12431
 atlas_init/cli_tf/schema_go_parser.py,sha256=PiRfFFVnkhltxcGFfOCgH53wwzIEynw2BXmSfaINLL8,8294
 atlas_init/cli_tf/schema_inspection.py,sha256=ujLvGfg3baByND4nRD0drZoI45STxo3VfYvim-PfVOc,1764
@@ -54,7 +56,7 @@ atlas_init/repos/go_sdk.py,sha256=1OzM9DjHEAzAAuI9ygoRRuhUK2gqpOhXExXRqhqa0tg,17
 atlas_init/repos/path.py,sha256=wrT8e01OBoAHj8iMrxqutgqWu-BHPe9-bEWtcZRu238,4187
 atlas_init/settings/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 atlas_init/settings/config.py,sha256=HIytZom8RRvpLGy6u8CpZ83tmFXI6v1tO3iSiuo08kc,6259
-atlas_init/settings/env_vars.py,sha256=q8Hj2LPJIg-PK0fCjrEigoPwTGIEbqjLEZckwgnkG8s,9688
+atlas_init/settings/env_vars.py,sha256=mfpQp5Ja5tdW_mUg5Nqneb294JehQAheWNOpHvDLbg0,9844
 atlas_init/settings/interactive.py,sha256=Xy1Z5WMAOSaJ-vQI_4xjAbSR92rWQgnffwVoDT27L68,340
 atlas_init/settings/path.py,sha256=KkXysu6-0AuSjsvYGknYGJX1hL2j1RD-Fpf8KsVYpkE,2618
 atlas_init/settings/rich_utils.py,sha256=5LgJUmc9wyJTsoS6xWKadrT0MoQREDaKvEOCuBLDXRg,1704
@@ -86,7 +88,7 @@ atlas_init/tf/modules/vpc_peering/vpc_peering.tf,sha256=hJ3KJdGbLpOQednUpVuiJ0Cq
 atlas_init/tf/modules/vpc_privatelink/atlas-privatelink.tf,sha256=FloaaX1MNDvoMZxBnEopeLKyfIlq6kaX2dmx8WWlXNU,1298
 atlas_init/tf/modules/vpc_privatelink/variables.tf,sha256=gktHCDYD4rz6CEpLg5aiXcFbugw4L5S2Fqc52QYdJyc,255
 atlas_init/tf/modules/vpc_privatelink/versions.tf,sha256=G0u5V_Hvvrkux_tqfOY05pA-GzSp_qILpfx1dZaTGDc,237
-atlas_init-0.3.1.dist-info/METADATA,sha256=vb-qsxAC6t4yI4ec_x1L-nYaGT2sGSt0LCIRHBPSgl8,5650
-atlas_init-0.3.1.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
-atlas_init-0.3.1.dist-info/entry_points.txt,sha256=oSNFIEAS9nUZyyZ8Fc-0F0U5j-NErygy01LpJVSHapQ,57
-atlas_init-0.3.1.dist-info/RECORD,,
+atlas_init-0.3.3.dist-info/METADATA,sha256=0BLRaxcC3IeB3umJ-rcNN3xRRrkspbSVMPOrLbmozhk,5650
+atlas_init-0.3.3.dist-info/WHEEL,sha256=C2FUgwZgiLbznR-k0b_5k3Ai_1aASOXDss3lzCUsUug,87
+atlas_init-0.3.3.dist-info/entry_points.txt,sha256=oSNFIEAS9nUZyyZ8Fc-0F0U5j-NErygy01LpJVSHapQ,57
+atlas_init-0.3.3.dist-info/RECORD,,